diff --git a/.github/workflows/ci.yml.disabled b/.github/workflows/ci.yml.disabled index 709c2a5d..07999587 100644 --- a/.github/workflows/ci.yml.disabled +++ b/.github/workflows/ci.yml.disabled @@ -76,6 +76,7 @@ jobs: - fastify-core - design-tokens - testing + - extraction steps: - name: Checkout @@ -124,6 +125,7 @@ jobs: - billing-service - growth-service - tracker-service + - extraction-service steps: - name: Checkout diff --git a/AGENTS.md b/AGENTS.md index cbf3b60d..2e7eaa02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,7 @@ learning_ai_common_plat/ │ ├── billing-service/ # Subscriptions, Stripe, usage, licenses (port 4002) │ ├── growth-service/ # Invitations, referrals, promos (port 4001) │ ├── tracker-service/ # Items, comments, votes, public roadmap (port 4004) +│ ├── extraction-service/ # LangExtract text extraction + Python sidecar (port 4005) │ └── monitoring/ # Loki + Grafana config, health-check script ├── docs/ # Architecture docs, roadmap, analysis ├── package.json # Root scripts: build, test, typecheck, clean @@ -105,34 +106,38 @@ learning_ai_common_plat/ ## 5. File Ownership Map -| Domain | Location | Key Files | -| ----------------------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| **Errors** | `packages/errors/` | `src/index.ts` — `BadRequestError`, `UnauthorizedError`, `ForbiddenError`, `NotFoundError`, `ConflictError`, `RateLimitError` | -| **Cosmos client** | `packages/cosmos/` | `src/index.ts` — `getCosmosClient()`, `getContainer()`, container registry | -| **Config / Product ID** | `packages/config/` | `src/index.ts` — `loadEnvConfig()`, `loadProductIdentity()`, `getProductId()` | -| **JWT / Auth** | `packages/auth/` | `src/index.ts` — `signJwt()`, `verifyJwt()`, `hashPassword()`, `verifyPassword()`, auth middleware | -| **API client** | `packages/api-client/` | `src/index.ts` — `createApiClient()` with token injection | -| **React auth** | `packages/react-auth/` | `src/index.ts` — `createAuthContext()` factory (provider + hook) | -| **Design tokens** | `packages/design-tokens/` | `tokens/bytelyst.tokens.json` (source), `scripts/generate.ts` (generator), `generated/` (output) | -| **Auth / JWT issue** | `services/platform-service/` | `src/modules/auth/` | -| **Feature flags** | `services/platform-service/` | `src/modules/flags/` — FNV-1a hash for deterministic rollout | -| **Blob storage** | `services/platform-service/` | `src/modules/blob/`, `src/lib/blob.ts` — SAS tokens, CRUD | -| **Audit log** | `services/platform-service/` | `src/modules/audit/` | -| **Notifications** | `services/platform-service/` | `src/modules/notifications/` | -| **Rate limiting** | `services/platform-service/` | `src/modules/rate-limit/` | -| **Subscriptions** | `services/billing-service/` | `src/modules/subscriptions/` | -| **Stripe webhooks** | `services/billing-service/` | `src/modules/stripe/` | -| **Usage tracking** | `services/billing-service/` | `src/modules/usage/` | -| **Plans** | `services/billing-service/` | `src/modules/plans/` | -| **Licenses** | `services/billing-service/` | `src/modules/licenses/` | -| **Invitations** | `services/growth-service/` | `src/modules/invitations/` | -| **Referrals** | `services/growth-service/` | `src/modules/referrals/` | -| **Promos** | `services/growth-service/` | `src/modules/promos/` | -| **Tracker items** | `services/tracker-service/` | `src/modules/items/` | -| **Public roadmap** | `services/tracker-service/` | `src/modules/public/` | -| **Tracker comments** | `services/tracker-service/` | `src/modules/comments/` | -| **Tracker votes** | `services/tracker-service/` | `src/modules/votes/` | -| **Monitoring** | `services/monitoring/` | `health-check.ts`, `loki/`, `grafana/` | +| Domain | Location | Key Files | +| ----------------------- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| **Errors** | `packages/errors/` | `src/index.ts` — `BadRequestError`, `UnauthorizedError`, `ForbiddenError`, `NotFoundError`, `ConflictError`, `RateLimitError` | +| **Cosmos client** | `packages/cosmos/` | `src/index.ts` — `getCosmosClient()`, `getContainer()`, container registry | +| **Config / Product ID** | `packages/config/` | `src/index.ts` — `loadEnvConfig()`, `loadProductIdentity()`, `getProductId()` | +| **JWT / Auth** | `packages/auth/` | `src/index.ts` — `signJwt()`, `verifyJwt()`, `hashPassword()`, `verifyPassword()`, auth middleware | +| **API client** | `packages/api-client/` | `src/index.ts` — `createApiClient()` with token injection | +| **React auth** | `packages/react-auth/` | `src/index.ts` — `createAuthContext()` factory (provider + hook) | +| **Design tokens** | `packages/design-tokens/` | `tokens/bytelyst.tokens.json` (source), `scripts/generate.ts` (generator), `generated/` (output) | +| **Auth / JWT issue** | `services/platform-service/` | `src/modules/auth/` | +| **Feature flags** | `services/platform-service/` | `src/modules/flags/` — FNV-1a hash for deterministic rollout | +| **Blob storage** | `services/platform-service/` | `src/modules/blob/`, `src/lib/blob.ts` — SAS tokens, CRUD | +| **Audit log** | `services/platform-service/` | `src/modules/audit/` | +| **Notifications** | `services/platform-service/` | `src/modules/notifications/` | +| **Rate limiting** | `services/platform-service/` | `src/modules/rate-limit/` | +| **Subscriptions** | `services/billing-service/` | `src/modules/subscriptions/` | +| **Stripe webhooks** | `services/billing-service/` | `src/modules/stripe/` | +| **Usage tracking** | `services/billing-service/` | `src/modules/usage/` | +| **Plans** | `services/billing-service/` | `src/modules/plans/` | +| **Licenses** | `services/billing-service/` | `src/modules/licenses/` | +| **Invitations** | `services/growth-service/` | `src/modules/invitations/` | +| **Referrals** | `services/growth-service/` | `src/modules/referrals/` | +| **Promos** | `services/growth-service/` | `src/modules/promos/` | +| **Tracker items** | `services/tracker-service/` | `src/modules/items/` | +| **Public roadmap** | `services/tracker-service/` | `src/modules/public/` | +| **Tracker comments** | `services/tracker-service/` | `src/modules/comments/` | +| **Tracker votes** | `services/tracker-service/` | `src/modules/votes/` | +| **Extraction routes** | `services/extraction-service/` | `src/modules/extract/` — POST /extract, /extract/batch, /extract/jobs, /extract/models | +| **Extraction tasks** | `services/extraction-service/` | `src/modules/tasks/` — predefined task library (triage, transcript, memory-insight, etc.) | +| **Extraction Python** | `services/extraction-service/` | `python/src/` — LangExtract sidecar (FastAPI :4006), extractor, task registry, language detection | +| **Extraction package** | `packages/extraction/` | `src/index.ts` — `createExtractionClient()`, shared types for consumers | +| **Monitoring** | `services/monitoring/` | `health-check.ts`, `loki/`, `grafana/` | ### Dashboard Consumers (via `file:` refs) @@ -270,7 +275,8 @@ DEFAULT_PRODUCT_ID=lysnrai @lysnrai/platform-service ← @bytelyst/{config, cosmos, errors} @lysnrai/billing-service ← @bytelyst/{config, cosmos, errors} @lysnrai/growth-service ← @bytelyst/{config, cosmos, errors} -@lysnrai/tracker-service ← @bytelyst/{config, cosmos, errors} +@lysnrai/tracker-service ← @bytelyst/{config, cosmos, errors} +@lysnrai/extraction-service ← @bytelyst/{fastify-core, config, cosmos, errors, auth} ``` Build order: packages first (they have no inter-deps), then services. `pnpm build` handles this automatically via workspace topology. @@ -288,13 +294,14 @@ Build order: packages first (they have no inter-deps), then services. `pnpm buil ## 11. Service Test Counts -| Service / Package | Tests | Runner | -| ----------------- | -------- | ------ | -| platform-service | 55 | vitest | -| tracker-service | 45 | vitest | -| growth-service | 33 | vitest | -| billing-service | 32 | vitest | -| **Total** | **165+** | | +| Service / Package | Tests | Runner | +| ------------------ | -------- | ------ | +| platform-service | 55 | vitest | +| tracker-service | 45 | vitest | +| extraction-service | 46 | vitest | +| growth-service | 33 | vitest | +| billing-service | 32 | vitest | +| **Total** | **211+** | | ## 12. Common Pitfalls diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a6446a6b..217edf49 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -537,7 +537,7 @@ The following gaps were identified by scanning every import in the actual codeba - [x] **7.6** Add `@bytelyst/testing` package (shared test utilities, mock factories) — created with 10 tests - [ ] **7.7** Evaluate Python shared package for `cosmos_client.py` + `blob_client.py` if MindLyst adds Python backend - [ ] **7.8** Integrate `@bytelyst/design-tokens` into LysnrAI dashboards (unified design language) -- [ ] **7.9** Add pre-commit hooks to auto-run token generation when JSON changes +- [x] **7.9** Add pre-commit hooks to auto-run token generation when JSON changes — lint-staged rule on `bytelyst.tokens.json` - [ ] **7.10** Set up Renovate/Dependabot for common-plat dependency updates --- @@ -557,8 +557,8 @@ The following gaps were identified by scanning every import in the actual codeba | **4** | `@bytelyst/design-tokens` (4 platforms) | 24 | 23 | ✅ CSS synced to MindLyst; CONTRIBUTING updated; visual verify pending | | **5** | CI/CD + Docker (pre-copy strategy) | 23 | 21 | ⚠️ All Dockerfiles rewritten, CI workflows created; Docker build blocked by proxy | | **6** | Verification + docs + cleanup | 28 | 21 | ✅ Docs updated, depcheck done, git clean; E2E needs services | -| **7** | Future enhancements (+testing pkg) | 10 | 3 | 🔲 @bytelyst/testing + @bytelyst/blob + @bytelyst/monitoring created | -| **Total** | **10 packages (+1 bonus: logger)** | **278** | **251** | **~90% complete** | +| **7** | Future enhancements (+testing pkg) | 10 | 3 | 🔲 @bytelyst/testing (10 tests) + token pre-commit hook + AGENTS updated | +| **Total** | **10 packages (+1 bonus: logger)** | **278** | **252** | **~91% complete** | ### Bonus Package (not in original roadmap) diff --git a/package.json b/package.json index 897670f5..7c133001 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,9 @@ ], "*.{js,jsx,json,md,yml,yaml}": [ "prettier --write" + ], + "packages/design-tokens/tokens/bytelyst.tokens.json": [ + "pnpm --filter @bytelyst/design-tokens generate" ] } }