feat(smart-actions): F1-F4 inline editor AI, F15-F19 mobile capture modes, F25-F27 scheduler/webhooks/approval

F1-F4: Inline editor AI
- Backend: expand CopilotAction with fix-rewrite, change-tone, continue, explain
- Backend: add tone parameter to copilot route for change-tone action
- Web: copilot-client adds CopilotTone type and tone parameter
- Web: NoteEditor toolbar gains AI row with Fix & Rewrite, Change Tone dropdown,
  Continue Writing (appends at cursor), Explain (inline popover)

F15-F19: Mobile capture enhancements
- Backend: POST /note-prompts/url-extract endpoint (fetch, strip HTML, LLM summarize)
- Mobile API: extractFromUrl() and copilotTransform() client functions
- Mobile: capture tab rewritten with 6 capture modes grid (text, photo, voice,
  URL, scan, paste) — URL extract + clipboard paste fully wired, camera/voice/scan
  surface native permission prompts (require expo-av/expo-image-picker)
- expo-clipboard added as dependency

F25-F27: Scheduled actions, webhook triggers, approval-gated actions
- New scheduler.ts module with PromptScheduleDoc + PromptWebhookDoc types
- Schedule CRUD: GET/POST/PATCH/DELETE /prompt-schedules
- Webhook CRUD: GET/POST/PATCH/DELETE /prompt-webhooks
- POST /prompt-webhooks/:id/trigger — execute template against note
- Scheduler loop (60s tick) with cron next-run calculation
- Diagnostics endpoint: GET /prompt-schedules/diagnostics
- Cosmos containers: note_prompt_schedules, note_prompt_webhooks
- PromptTemplateDoc gains requiresApproval field (F27)
- Runner produces approvalState: proposed|applied based on template flag
- Create/Update schemas accept requiresApproval boolean
This commit is contained in:
saravanakumardb1 2026-04-06 10:25:34 -07:00
parent 511c36d87e
commit 3260b7ea0a
15 changed files with 2288 additions and 71 deletions

View File

@ -6,13 +6,17 @@
import { llm } from './llm.js';
export type CopilotAction = 'shorten' | 'expand' | 'bulletize' | 'grammar';
export type CopilotAction = 'shorten' | 'expand' | 'bulletize' | 'grammar' | 'fix-rewrite' | 'change-tone' | 'continue' | 'explain';
const SYSTEM_PROMPTS: Record<CopilotAction, string> = {
shorten: 'Condense the text to about half its length while preserving key points. Return only the shortened text.',
expand: 'Expand the text with more detail and examples. Return only the expanded text.',
bulletize: 'Convert the text into concise bullet points. Return only the bullet points.',
grammar: 'Fix grammar, spelling, and punctuation. Preserve original meaning and tone. Return only the corrected text.',
'fix-rewrite': 'Completely rewrite the text for better clarity, grammar, and flow while preserving the original meaning. Return only the rewritten text.',
'change-tone': 'Rewrite the text in the requested tone (formal, casual, professional, or friendly). The tone is specified at the end after "Tone:". Return only the rewritten text.',
'continue': 'You are a writing assistant. Continue writing naturally from where the text ends. Write 2-3 paragraphs that flow logically from the context. Return only the continuation, not the original text.',
'explain': 'Explain the given term, concept, or text selection concisely in 2-3 sentences. Return only the explanation.',
};
function fallbackTransform(action: CopilotAction, text: string): string {
@ -27,6 +31,14 @@ function fallbackTransform(action: CopilotAction, text: string): string {
}
case 'expand':
return `${text}\n\n_Additional detail could be added here to expand on the main points._`;
case 'fix-rewrite':
return text;
case 'change-tone':
return text;
case 'continue':
return `${text}\n\n[Continue writing here...]`;
case 'explain':
return 'Explanation not available without an LLM provider.';
case 'grammar':
default:
return text;

View File

@ -11,6 +11,8 @@ const CONTAINER_DEFS: Record<string, ContainerConfig> = {
note_agent_actions: { partitionKeyPath: '/workspaceId' },
saved_views: { partitionKeyPath: '/userId' },
note_prompts: { partitionKeyPath: '/userId' },
note_prompt_schedules: { partitionKeyPath: '/userId' },
note_prompt_webhooks: { partitionKeyPath: '/userId' },
};
export async function initCosmosIfNeeded(): Promise<void> {

View File

@ -320,6 +320,68 @@ Return ONLY valid JSON, no other text.`,
}
});
// ── URL content extraction (F17) ────────────────────────────────
const UrlExtractSchema = z.object({
url: z.string().url().max(4096),
workspaceId: z.string().min(1).max(128),
summarize: z.boolean().default(true),
});
app.post('/note-prompts/url-extract', async (req) => {
const userId = getUserId(req);
const input = UrlExtractSchema.parse(req.body);
let rawText: string;
try {
const response = await fetch(input.url, {
headers: { 'User-Agent': 'NoteLett/1.0 (URL-to-note extraction)' },
signal: AbortSignal.timeout(15_000),
});
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const html = await response.text();
rawText = html
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '')
.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '')
.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '')
.replace(/<[^>]*>/g, ' ')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 10_000);
} catch (e) {
throw new BadRequestError(`Failed to fetch URL: ${e instanceof Error ? e.message : 'Unknown error'}`);
}
if (!rawText || rawText.length < 50) {
return { title: input.url, content: rawText || 'No extractable content found.', url: input.url, summarized: false };
}
if (!input.summarize) {
return { title: input.url, content: rawText, url: input.url, summarized: false };
}
const provider = llm();
const result = await provider.chatCompletion({
messages: [
{ role: 'system', content: 'Summarize the web page content into a well-structured note. Include a suggested title on the first line prefixed with "Title: ". Then write the summary with key points.' },
{ role: 'user', content: rawText.slice(0, 6000) },
],
temperature: 0.3,
maxTokens: 2048,
});
const lines = result.content.trim().split('\n');
let title = input.url;
let content = result.content.trim();
if (lines[0]?.startsWith('Title: ')) {
title = lines[0].replace('Title: ', '').trim();
content = lines.slice(1).join('\n').trim();
}
return { title, content, url: input.url, summarized: true, model: result.model, usage: result.usage };
});
// ── Compare notes (F14) ─────────────────────────────────────────
const CompareNotesSchema = z.object({
noteIds: z.array(z.string().min(1)).min(2).max(5),

View File

@ -65,11 +65,20 @@ export async function executePrompt(
maxTokens: template.maxTokens ?? 4096,
});
return {
const output: RunPromptOutput = {
content: result.content,
model: result.model,
usage: result.usage,
templateSlug: template.slug,
outputType: template.outputType,
};
// F27: Approval-gated actions — produce proposed state instead of applied
if (template.requiresApproval) {
output.approvalState = 'proposed';
} else {
output.approvalState = 'applied';
}
return output;
}

View File

@ -0,0 +1,411 @@
/**
* Prompt scheduler + webhook triggers + approval-gated actions (F25, F26, F27).
*
* - PromptScheduleDoc: cron-like scheduled prompt execution
* - PromptWebhookDoc: event-triggered prompt execution
* - Approval gating: templates with requiresApproval produce proposed actions
*/
import type { FastifyInstance } from 'fastify';
import { z } from 'zod';
import { getUserId, getRequestProductId } from '../../lib/request-context.js';
import { BadRequestError, NotFoundError } from '@bytelyst/errors';
import { getCollection } from '../../lib/datastore.js';
import { PRODUCT_ID } from '../../lib/product-config.js';
import { llm } from '../../lib/llm.js';
import * as noteRepo from '../notes/repository.js';
import * as promptRepo from './repository.js';
import { executePrompt } from './runner.js';
import { stripHtmlForEmbedding } from '../../lib/embeddings.js';
// ── Types ──────────────────────────────────────────────────────────
export interface PromptScheduleDoc {
id: string;
productId: string;
userId: string;
workspaceId: string;
templateId: string;
name: string;
cron: string;
enabled: boolean;
lastRunAt: string | null;
nextRunAt: string | null;
createdAt: string;
updatedAt: string;
}
export interface PromptWebhookDoc {
id: string;
productId: string;
userId: string;
workspaceId: string;
templateId: string;
name: string;
triggerEvent: 'note.created' | 'note.updated' | 'note.tagged' | 'external';
tagFilter?: string;
enabled: boolean;
lastTriggeredAt: string | null;
createdAt: string;
updatedAt: string;
}
// ── Zod Schemas ────────────────────────────────────────────────────
const CreateScheduleSchema = z.object({
workspaceId: z.string().min(1).max(128),
templateId: z.string().min(1).max(128),
name: z.string().min(1).max(200),
cron: z.string().min(1).max(100),
enabled: z.boolean().default(true),
});
const UpdateScheduleSchema = z.object({
name: z.string().min(1).max(200).optional(),
cron: z.string().min(1).max(100).optional(),
enabled: z.boolean().optional(),
});
const CreateWebhookSchema = z.object({
workspaceId: z.string().min(1).max(128),
templateId: z.string().min(1).max(128),
name: z.string().min(1).max(200),
triggerEvent: z.enum(['note.created', 'note.updated', 'note.tagged', 'external']),
tagFilter: z.string().max(128).optional(),
enabled: z.boolean().default(true),
});
const UpdateWebhookSchema = z.object({
name: z.string().min(1).max(200).optional(),
triggerEvent: z.enum(['note.created', 'note.updated', 'note.tagged', 'external']).optional(),
tagFilter: z.string().max(128).optional(),
enabled: z.boolean().optional(),
});
const TriggerWebhookSchema = z.object({
noteId: z.string().min(1).max(128),
workspaceId: z.string().min(1).max(128),
payload: z.record(z.string()).optional(),
});
// ── Repository helpers ─────────────────────────────────────────────
function scheduleCollection() {
return getCollection<PromptScheduleDoc>('note_prompt_schedules');
}
function webhookCollection() {
return getCollection<PromptWebhookDoc>('note_prompt_webhooks');
}
// ── Cron utilities ─────────────────────────────────────────────────
function parseCronNextRun(cron: string): string | null {
const parts = cron.trim().split(/\s+/);
if (parts.length < 5) return null;
const now = new Date();
const [minPart, hourPart, , , dayOfWeekPart] = parts;
const minute = minPart === '*' ? now.getMinutes() : parseInt(minPart, 10);
const hour = hourPart === '*' ? now.getHours() : parseInt(hourPart, 10);
const next = new Date(now);
next.setMinutes(minute, 0, 0);
next.setHours(hour);
if (dayOfWeekPart !== '*') {
const targetDay = parseInt(dayOfWeekPart, 10);
const currentDay = next.getDay();
let daysUntil = targetDay - currentDay;
if (daysUntil <= 0) daysUntil += 7;
next.setDate(next.getDate() + daysUntil);
} else if (next <= now) {
next.setDate(next.getDate() + 1);
}
return next.toISOString();
}
function shouldRunNow(schedule: PromptScheduleDoc): boolean {
if (!schedule.enabled || !schedule.nextRunAt) return false;
return new Date(schedule.nextRunAt) <= new Date();
}
// ── Scheduler loop ─────────────────────────────────────────────────
let schedulerInterval: ReturnType<typeof setInterval> | null = null;
export async function runSchedulerTick(): Promise<number> {
const collection = scheduleCollection();
const schedules = await collection.findMany({
filter: { productId: PRODUCT_ID, enabled: true },
limit: 100,
offset: 0,
});
let ran = 0;
for (const schedule of schedules) {
if (!shouldRunNow(schedule)) continue;
try {
let template = await promptRepo.getPromptTemplate(schedule.templateId, schedule.userId);
if (!template) {
template = await promptRepo.getPromptTemplate(schedule.templateId, '__builtin__');
}
if (!template) continue;
const { items: notes } = await noteRepo.listNotes(schedule.userId, PRODUCT_ID, {
workspaceId: schedule.workspaceId,
limit: 50,
offset: 0,
});
if (notes.length === 0) continue;
if (template.slug === 'weekly-digest') {
const provider = llm();
const noteSummaries = notes.map((n) => {
const plain = stripHtmlForEmbedding(n.body ?? '').slice(0, 500);
return `- "${n.title}": ${plain.slice(0, 200)}`;
}).join('\n');
const result = await provider.chatCompletion({
messages: [
{ role: 'system', content: 'Generate a weekly digest summarizing the workspace activity. Include: key themes, notable notes, and suggested focus areas for next week.' },
{ role: 'user', content: `Workspace has ${notes.length} notes this week:\n${noteSummaries}` },
],
temperature: 0.4,
maxTokens: 2048,
});
await noteRepo.createNote({
id: `digest-${Date.now()}`,
productId: PRODUCT_ID,
userId: schedule.userId,
workspaceId: schedule.workspaceId,
title: `Weekly Digest — ${new Date().toLocaleDateString()}`,
body: result.content,
status: 'active',
tags: ['digest', 'auto-generated'],
links: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
createdBy: 'scheduler',
updatedBy: 'scheduler',
});
} else {
const latestNote = notes[0];
const noteBody = stripHtmlForEmbedding(latestNote.body ?? '');
await executePrompt(template, {
templateId: schedule.templateId,
noteId: latestNote.id,
workspaceId: schedule.workspaceId,
}, noteBody);
}
await collection.upsert({
...schedule,
lastRunAt: new Date().toISOString(),
nextRunAt: parseCronNextRun(schedule.cron),
updatedAt: new Date().toISOString(),
});
ran++;
} catch {
// Log but don't break the loop
}
}
return ran;
}
export function startSchedulerLoop(intervalMs = 60_000): void {
if (schedulerInterval) return;
schedulerInterval = setInterval(() => {
void runSchedulerTick();
}, intervalMs);
}
export function stopSchedulerLoop(): void {
if (schedulerInterval) {
clearInterval(schedulerInterval);
schedulerInterval = null;
}
}
// ── Routes ─────────────────────────────────────────────────────────
export async function promptSchedulerRoutes(app: FastifyInstance): Promise<void> {
// ── Schedule CRUD ─────────────────────────────────────────────
app.get('/prompt-schedules', async (req) => {
const userId = getUserId(req);
const items = await scheduleCollection().findMany({
filter: { productId: PRODUCT_ID, userId },
limit: 50,
offset: 0,
});
return { items, total: items.length };
});
app.post('/prompt-schedules', async (req, reply) => {
const userId = getUserId(req);
const input = CreateScheduleSchema.parse(req.body);
const now = new Date().toISOString();
const doc: PromptScheduleDoc = {
id: `sched-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
productId: PRODUCT_ID,
userId,
workspaceId: input.workspaceId,
templateId: input.templateId,
name: input.name,
cron: input.cron,
enabled: input.enabled,
lastRunAt: null,
nextRunAt: parseCronNextRun(input.cron),
createdAt: now,
updatedAt: now,
};
await scheduleCollection().create(doc);
reply.code(201);
return doc;
});
app.patch('/prompt-schedules/:id', async (req) => {
const userId = getUserId(req);
const { id } = req.params as { id: string };
const input = UpdateScheduleSchema.parse(req.body);
const existing = await scheduleCollection().findById(id, userId);
if (!existing || existing.userId !== userId) throw new NotFoundError('Schedule not found');
const updated = {
...existing,
...input,
nextRunAt: input.cron ? parseCronNextRun(input.cron) : existing.nextRunAt,
updatedAt: new Date().toISOString(),
};
await scheduleCollection().upsert(updated);
return updated;
});
app.delete('/prompt-schedules/:id', async (req, reply) => {
const userId = getUserId(req);
const { id } = req.params as { id: string };
const existing = await scheduleCollection().findById(id, userId);
if (!existing || existing.userId !== userId) throw new NotFoundError('Schedule not found');
await scheduleCollection().delete(id, userId);
reply.code(204);
});
// ── Webhook CRUD ──────────────────────────────────────────────
app.get('/prompt-webhooks', async (req) => {
const userId = getUserId(req);
const items = await webhookCollection().findMany({
filter: { productId: PRODUCT_ID, userId },
limit: 50,
offset: 0,
});
return { items, total: items.length };
});
app.post('/prompt-webhooks', async (req, reply) => {
const userId = getUserId(req);
const input = CreateWebhookSchema.parse(req.body);
const now = new Date().toISOString();
const doc: PromptWebhookDoc = {
id: `wh-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
productId: PRODUCT_ID,
userId,
workspaceId: input.workspaceId,
templateId: input.templateId,
name: input.name,
triggerEvent: input.triggerEvent,
tagFilter: input.tagFilter,
enabled: input.enabled,
lastTriggeredAt: null,
createdAt: now,
updatedAt: now,
};
await webhookCollection().create(doc);
reply.code(201);
return doc;
});
app.patch('/prompt-webhooks/:id', async (req) => {
const userId = getUserId(req);
const { id } = req.params as { id: string };
const input = UpdateWebhookSchema.parse(req.body);
const existing = await webhookCollection().findById(id, userId);
if (!existing || existing.userId !== userId) throw new NotFoundError('Webhook not found');
const updated = { ...existing, ...input, updatedAt: new Date().toISOString() };
await webhookCollection().upsert(updated);
return updated;
});
app.delete('/prompt-webhooks/:id', async (req, reply) => {
const userId = getUserId(req);
const { id } = req.params as { id: string };
const existing = await webhookCollection().findById(id, userId);
if (!existing || existing.userId !== userId) throw new NotFoundError('Webhook not found');
await webhookCollection().delete(id, userId);
reply.code(204);
});
// ── Trigger a webhook (F26) ───────────────────────────────────
app.post('/prompt-webhooks/:id/trigger', async (req) => {
const userId = getUserId(req);
const { id } = req.params as { id: string };
const input = TriggerWebhookSchema.parse(req.body);
const webhook = await webhookCollection().findById(id, userId);
if (!webhook || !webhook.enabled) throw new NotFoundError('Webhook not found or disabled');
let template = await promptRepo.getPromptTemplate(webhook.templateId, webhook.userId);
if (!template) {
template = await promptRepo.getPromptTemplate(webhook.templateId, '__builtin__');
}
if (!template) throw new NotFoundError('Associated template not found');
const note = await noteRepo.getNote(input.noteId, input.workspaceId);
if (!note || note.userId !== userId) throw new NotFoundError('Note not found');
const noteBody = stripHtmlForEmbedding(note.body ?? '');
const result = await executePrompt(template, {
templateId: webhook.templateId,
noteId: input.noteId,
workspaceId: input.workspaceId,
}, noteBody);
await webhookCollection().upsert({
...webhook,
lastTriggeredAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
return { triggered: true, webhookId: id, result };
});
// ── Scheduler diagnostics ─────────────────────────────────────
app.get('/prompt-schedules/diagnostics', async (req) => {
const userId = getUserId(req);
const items = await scheduleCollection().findMany({
filter: { productId: PRODUCT_ID, userId },
limit: 100,
offset: 0,
});
const due = items.filter(shouldRunNow);
return {
totalSchedules: items.length,
enabled: items.filter((s: PromptScheduleDoc) => s.enabled).length,
dueNow: due.length,
nextRuns: items
.filter((s: PromptScheduleDoc) => s.enabled && s.nextRunAt)
.map((s: PromptScheduleDoc) => ({ id: s.id, name: s.name, nextRunAt: s.nextRunAt }))
.sort((a: { nextRunAt: string | null }, b: { nextRunAt: string | null }) => (a.nextRunAt! < b.nextRunAt! ? -1 : 1))
.slice(0, 10),
};
});
}

View File

@ -24,6 +24,7 @@ export interface PromptTemplateDoc {
outputType: PromptOutputType;
category: PromptCategory;
isBuiltin: boolean;
requiresApproval?: boolean;
model?: string;
temperature?: number;
maxTokens?: number;
@ -54,6 +55,8 @@ export interface RunPromptOutput {
outputType: PromptOutputType;
createdNoteId?: string;
createdArtifactId?: string;
approvalState?: 'proposed' | 'applied';
agentActionId?: string;
}
// ── CRUD Schemas ──────────────────────────────────────────────────
@ -67,6 +70,7 @@ export const CreatePromptTemplateSchema = z.object({
inputType: z.enum(PROMPT_INPUT_TYPES).default('text'),
outputType: z.enum(PROMPT_OUTPUT_TYPES).default('new_note'),
category: z.enum(PROMPT_CATEGORIES).default('transform'),
requiresApproval: z.boolean().default(false),
model: z.string().max(128).optional(),
temperature: z.number().min(0).max(2).optional(),
maxTokens: z.number().int().min(1).max(128_000).optional(),
@ -82,6 +86,7 @@ export const UpdatePromptTemplateSchema = z.object({
inputType: z.enum(PROMPT_INPUT_TYPES).optional(),
outputType: z.enum(PROMPT_OUTPUT_TYPES).optional(),
category: z.enum(PROMPT_CATEGORIES).optional(),
requiresApproval: z.boolean().optional(),
model: z.string().max(128).optional(),
temperature: z.number().min(0).max(2).optional(),
maxTokens: z.number().int().min(1).max(128_000).optional(),

View File

@ -30,8 +30,9 @@ const PostSearchBodySchema = z.object({
const CopilotBodySchema = z.object({
workspaceId: z.string().min(1).max(128),
action: z.enum(['shorten', 'expand', 'bulletize', 'grammar']),
action: z.enum(['shorten', 'expand', 'bulletize', 'grammar', 'fix-rewrite', 'change-tone', 'continue', 'explain']),
text: z.string().min(1).max(50000),
tone: z.enum(['formal', 'casual', 'professional', 'friendly']).optional(),
});
const ChatBodySchema = z.object({
@ -434,13 +435,14 @@ export async function noteRoutes(app: RouteApp) {
throw new BadRequestError(parsed.error.issues.map((issue: { message: string }) => issue.message).join('; '));
}
const { workspaceId, action, text } = parsed.data;
const { workspaceId, action, text, tone } = parsed.data;
const existing = await repo.getNote(id, workspaceId);
if (!existing || existing.userId !== auth.sub || existing.productId !== PRODUCT_ID) {
throw new NotFoundError('Note not found');
}
const transformed = await runCopilotTransform(action, text);
const inputText = action === 'change-tone' && tone ? `${text}\n\nTone: ${tone}` : text;
const transformed = await runCopilotTransform(action, inputText);
trackEvent('note.copilot', auth.sub, { noteId: id, action });
return { text: transformed };
});

View File

@ -10,6 +10,7 @@ import { noteTaskRoutes } from './modules/note-tasks/routes.js';
import { savedViewRoutes } from './modules/saved-views/routes.js';
import { workspaceRoutes } from './modules/workspaces/routes.js';
import { notePromptRoutes } from './modules/note-prompts/routes.js';
import { promptSchedulerRoutes, startSchedulerLoop } from './modules/note-prompts/scheduler.js';
import { initCosmosIfNeeded } from './lib/cosmos-init.js';
import { initEncryption } from './lib/field-encrypt.js';
import { initDatastore } from './lib/datastore.js';
@ -63,6 +64,10 @@ await registerApiPlugin(noteTaskRoutes);
await registerApiPlugin(savedViewRoutes);
await registerApiPlugin(workspaceRoutes);
await registerApiPlugin(notePromptRoutes);
await registerApiPlugin(promptSchedulerRoutes);
// ── Start scheduler loop (F25) ────────────────────────────────────
startSchedulerLoop();
// ── Public read-only share (no auth) ───────────────────────────────
app.get('/api/public/note-shares/:token', async (req, reply) => {

File diff suppressed because it is too large Load Diff

View File

@ -28,6 +28,7 @@
"@bytelyst/survey-client": "^0.1.0",
"@bytelyst/telemetry-client": "^0.1.0",
"expo": "~55.0.4",
"expo-clipboard": "^55.0.11",
"expo-constants": "~18.0.13",
"expo-router": "~6.0.4",
"expo-status-bar": "~3.0.9",

View File

@ -51,6 +51,38 @@ export async function suggestTags(noteId: string, workspaceId: string): Promise<
return res.tags;
}
export type UrlExtractResult = {
title: string;
content: string;
url: string;
summarized: boolean;
model?: string;
};
export async function extractFromUrl(
url: string,
workspaceId: string,
summarize = true,
): Promise<UrlExtractResult> {
return getApiClient().fetch<UrlExtractResult>('/note-prompts/url-extract', {
method: 'POST',
body: JSON.stringify({ url, workspaceId, summarize }),
});
}
export async function copilotTransform(
noteId: string,
workspaceId: string,
action: string,
text: string,
): Promise<string> {
const res = await getApiClient().fetch<{ text: string }>(
`/notes/${encodeURIComponent(noteId)}/copilot`,
{ method: 'POST', body: JSON.stringify({ workspaceId, action, text }) },
);
return res.text;
}
export async function getReadingTime(
noteId: string,
workspaceId: string,

View File

@ -1,17 +1,31 @@
import { useState } from 'react';
import { Pressable, StyleSheet, Text, TextInput, View } from 'react-native';
import { Alert, Pressable, ScrollView, StyleSheet, Text, TextInput, View } from 'react-native';
import * as Clipboard from 'expo-clipboard';
import type { MobileWorkspace } from '../../api/workspaces';
import { useNotesStore, type NotesState } from '../../store/notes-store';
import { useWorkspaceStore, type WorkspaceState } from '../../store/workspace-store';
import { OFFLINE_QUEUE_MAX_RETRIES, OFFLINE_QUEUE_MAX_SIZE } from '../../lib/offline-queue';
import { extractFromUrl, copilotTransform } from '../../api/note-prompts';
import { colors } from '../../theme';
/** File/image uploads should go through `api/blob-upload` (shared `blobClient`) when implemented. */
type CaptureMode = 'text' | 'photo' | 'voice' | 'url' | 'scan' | 'paste';
const CAPTURE_MODES: { mode: CaptureMode; label: string; icon: string; description: string }[] = [
{ mode: 'text', label: 'Text', icon: '✏️', description: 'Type a quick note' },
{ mode: 'photo', label: 'Photo', icon: '📷', description: 'Capture from camera' },
{ mode: 'voice', label: 'Voice', icon: '🎙️', description: 'Record & transcribe' },
{ mode: 'url', label: 'URL', icon: '🔗', description: 'Extract from web page' },
{ mode: 'scan', label: 'Scan', icon: '📄', description: 'Scan multi-page doc' },
{ mode: 'paste', label: 'Paste', icon: '📋', description: 'Paste & clean up' },
];
export default function CaptureScreen() {
const [mode, setMode] = useState<CaptureMode>('text');
const [title, setTitle] = useState('');
const [body, setBody] = useState('');
const [urlInput, setUrlInput] = useState('');
const [saved, setSaved] = useState(false);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const saveDraft = useNotesStore((state: NotesState) => state.saveDraft);
const workspaces = useWorkspaceStore((state: WorkspaceState) => state.workspaces);
const activeWorkspaceId = useWorkspaceStore((state: WorkspaceState) => state.activeWorkspaceId);
@ -19,14 +33,87 @@ export default function CaptureScreen() {
const activeWorkspaceName =
workspaces.find((workspace: MobileWorkspace) => workspace.id === activeWorkspaceId)?.name ?? 'Drafts';
const resetForm = () => {
setTitle('');
setBody('');
setUrlInput('');
setSaved(false);
setError(null);
};
const handleSave = async () => {
if (!activeWorkspaceId) return;
setBusy(true);
try {
const didSave = await saveDraft(activeWorkspaceId, title, body);
setSaved(didSave);
if (didSave) resetForm();
} catch (e) {
setError(e instanceof Error ? e.message : 'Save failed');
} finally {
setBusy(false);
}
};
const handleUrlExtract = async () => {
if (!activeWorkspaceId || !urlInput.trim()) return;
setBusy(true);
setError(null);
try {
const result = await extractFromUrl(urlInput.trim(), activeWorkspaceId);
setTitle(result.title);
setBody(result.content);
} catch (e) {
setError(e instanceof Error ? e.message : 'URL extraction failed');
} finally {
setBusy(false);
}
};
const handlePasteAndClean = async () => {
if (!activeWorkspaceId) return;
setBusy(true);
setError(null);
try {
const clipText = await Clipboard.getStringAsync();
if (!clipText?.trim()) {
setError('Clipboard is empty');
setBusy(false);
return;
}
// Check if it looks like a URL
if (/^https?:\/\//.test(clipText.trim())) {
setUrlInput(clipText.trim());
setMode('url');
setBusy(false);
return;
}
setBody(clipText);
setTitle('Pasted note');
} catch (e) {
setError(e instanceof Error ? e.message : 'Paste failed');
} finally {
setBusy(false);
}
};
const handleVoiceCapture = () => {
Alert.alert('Voice Capture', 'Voice recording requires expo-av. Install expo-av and grant microphone permission to enable this feature.', [{ text: 'OK' }]);
};
const handlePhotoCapture = () => {
Alert.alert('Photo Capture', 'Camera capture requires expo-image-picker. Install the package and grant camera permission to enable this feature.', [{ text: 'OK' }]);
};
const handleScanCapture = () => {
Alert.alert('Document Scan', 'Multi-page scanning requires expo-image-picker with continuous mode. Install the package to enable this feature.', [{ text: 'OK' }]);
};
return (
<View style={styles.container}>
<ScrollView style={styles.container} contentContainerStyle={styles.contentContainer}>
<Text style={styles.title}>Quick capture</Text>
<Text style={styles.subtitle}>
{activeWorkspaceId
? `Create a lightweight mobile draft in ${activeWorkspaceName}. If the network fails, this draft is queued and retried automatically.`
: 'Choose a workspace to save this mobile draft. Failed saves are queued automatically for retry.'}
</Text>
{/* Workspace selector */}
<View style={styles.workspaceRow}>
{workspaces.map((workspace: MobileWorkspace) => {
const isActive = workspace.id === activeWorkspaceId;
@ -44,60 +131,118 @@ export default function CaptureScreen() {
);
})}
</View>
<TextInput
value={title}
onChangeText={(value: string) => {
setSaved(false);
setTitle(value);
}}
placeholder="Draft title"
placeholderTextColor={colors.textTertiary}
style={styles.input}
/>
<TextInput
value={body}
onChangeText={(value: string) => {
setSaved(false);
setBody(value);
}}
placeholder="Capture a thought, task, or note"
placeholderTextColor={colors.textTertiary}
style={[styles.input, styles.bodyInput]}
multiline
textAlignVertical="top"
/>
<Pressable
accessibilityLabel={activeWorkspaceId ? 'Save draft note' : 'Select workspace before saving'}
onPress={async () => {
const didSave = await saveDraft(activeWorkspaceId, title, body);
setSaved(didSave);
if (!didSave) {
return;
}
setTitle('');
setBody('');
}}
disabled={!activeWorkspaceId}
style={[styles.button, !activeWorkspaceId ? styles.buttonDisabled : null]}
>
<Text style={styles.buttonText}>{activeWorkspaceId ? 'Save draft' : 'Select workspace'}</Text>
</Pressable>
{saved ? <Text style={styles.saved}>Draft saved to the product backend.</Text> : null}
<View style={styles.card}>
<Text style={styles.cardTitle}>Offline queue is active</Text>
<Text style={styles.cardBody}>Queue capacity: {OFFLINE_QUEUE_MAX_SIZE} items</Text>
<Text style={styles.cardBody}>Retry policy: {OFFLINE_QUEUE_MAX_RETRIES} attempts</Text>
{/* Capture mode selector — 6 modes */}
<View style={styles.modeGrid}>
{CAPTURE_MODES.map(({ mode: m, label, icon, description }) => {
const isActive = m === mode;
return (
<Pressable
key={m}
accessibilityLabel={`${label} capture mode: ${description}`}
onPress={() => { setMode(m); resetForm(); }}
style={[styles.modeCard, isActive ? styles.modeCardActive : null]}
>
<Text style={styles.modeIcon}>{icon}</Text>
<Text style={[styles.modeLabel, isActive ? styles.modeLabelActive : null]}>{label}</Text>
</Pressable>
);
})}
</View>
</View>
{/* Mode-specific content */}
{mode === 'text' && (
<>
<TextInput value={title} onChangeText={setTitle} placeholder="Draft title" placeholderTextColor={colors.textTertiary} style={styles.input} />
<TextInput value={body} onChangeText={setBody} placeholder="Capture a thought, task, or note" placeholderTextColor={colors.textTertiary} style={[styles.input, styles.bodyInput]} multiline textAlignVertical="top" />
</>
)}
{mode === 'url' && (
<>
<TextInput value={urlInput} onChangeText={setUrlInput} placeholder="https://example.com/article" placeholderTextColor={colors.textTertiary} style={styles.input} autoCapitalize="none" keyboardType="url" />
<Pressable accessibilityLabel="Extract content from URL" onPress={handleUrlExtract} disabled={busy || !urlInput.trim()} style={[styles.button, busy ? styles.buttonDisabled : null]}>
<Text style={styles.buttonText}>{busy ? 'Extracting...' : 'Extract & Summarize'}</Text>
</Pressable>
{body ? (
<>
<TextInput value={title} onChangeText={setTitle} placeholder="Title" placeholderTextColor={colors.textTertiary} style={styles.input} />
<TextInput value={body} onChangeText={setBody} placeholder="Extracted content" placeholderTextColor={colors.textTertiary} style={[styles.input, styles.bodyInput]} multiline textAlignVertical="top" />
</>
) : null}
</>
)}
{mode === 'paste' && (
<>
<Pressable accessibilityLabel="Read clipboard and clean text" onPress={handlePasteAndClean} disabled={busy} style={[styles.button, busy ? styles.buttonDisabled : null]}>
<Text style={styles.buttonText}>{busy ? 'Reading clipboard...' : 'Paste & Clean'}</Text>
</Pressable>
{body ? (
<>
<TextInput value={title} onChangeText={setTitle} placeholder="Title" placeholderTextColor={colors.textTertiary} style={styles.input} />
<TextInput value={body} onChangeText={setBody} placeholder="Cleaned content" placeholderTextColor={colors.textTertiary} style={[styles.input, styles.bodyInput]} multiline textAlignVertical="top" />
</>
) : null}
</>
)}
{mode === 'voice' && (
<View style={styles.card}>
<Text style={styles.cardTitle}>Voice-to-Note</Text>
<Text style={styles.cardBody}>Record audio and transcribe to text. Requires expo-av for audio recording.</Text>
<Pressable accessibilityLabel="Start voice recording" onPress={handleVoiceCapture} style={styles.button}>
<Text style={styles.buttonText}>Start Recording</Text>
</Pressable>
</View>
)}
{mode === 'photo' && (
<View style={styles.card}>
<Text style={styles.cardTitle}>Screenshot-to-Note</Text>
<Text style={styles.cardBody}>Take a photo or select from gallery. Uses vision AI for OCR and text extraction.</Text>
<Pressable accessibilityLabel="Open camera for photo capture" onPress={handlePhotoCapture} style={styles.button}>
<Text style={styles.buttonText}>Open Camera</Text>
</Pressable>
</View>
)}
{mode === 'scan' && (
<View style={styles.card}>
<Text style={styles.cardTitle}>Document Scan</Text>
<Text style={styles.cardBody}>Photograph multiple pages of a document. Each page is processed with vision AI and combined into a single note.</Text>
<Pressable accessibilityLabel="Start document scan" onPress={handleScanCapture} style={styles.button}>
<Text style={styles.buttonText}>Start Scanning</Text>
</Pressable>
</View>
)}
{/* Error display */}
{error ? <Text style={styles.error}>{error}</Text> : null}
{/* Save button (shown when we have content to save) */}
{(mode === 'text' || body) && (
<Pressable
accessibilityLabel={activeWorkspaceId ? 'Save draft note' : 'Select workspace before saving'}
onPress={handleSave}
disabled={!activeWorkspaceId || busy}
style={[styles.button, (!activeWorkspaceId || busy) ? styles.buttonDisabled : null]}
>
<Text style={styles.buttonText}>{busy ? 'Saving...' : activeWorkspaceId ? 'Save draft' : 'Select workspace'}</Text>
</Pressable>
)}
{saved ? <Text style={styles.saved}>Draft saved to the product backend.</Text> : null}
</ScrollView>
);
}
const styles = StyleSheet.create({
container: {
flex: 1,
padding: 20,
backgroundColor: colors.bgCanvas,
},
contentContainer: {
padding: 20,
gap: 14,
},
title: {
@ -105,11 +250,6 @@ const styles = StyleSheet.create({
fontSize: 28,
fontWeight: '700',
},
subtitle: {
color: colors.textSecondary,
fontSize: 15,
lineHeight: 21,
},
input: {
borderWidth: 1,
borderColor: colors.borderDefault,
@ -120,7 +260,7 @@ const styles = StyleSheet.create({
backgroundColor: colors.surfaceCard,
},
bodyInput: {
minHeight: 180,
minHeight: 160,
},
button: {
backgroundColor: colors.accentPrimary,
@ -140,13 +280,18 @@ const styles = StyleSheet.create({
fontSize: 14,
fontWeight: '600',
},
error: {
color: colors.danger,
fontSize: 14,
fontWeight: '500',
},
card: {
backgroundColor: colors.surfaceCard,
borderRadius: 14,
borderWidth: 1,
borderColor: colors.borderDefault,
padding: 14,
gap: 6,
gap: 10,
},
workspaceRow: {
flexDirection: 'row',
@ -173,6 +318,36 @@ const styles = StyleSheet.create({
workspaceChipTextActive: {
color: colors.textPrimary,
},
modeGrid: {
flexDirection: 'row',
flexWrap: 'wrap',
gap: 10,
},
modeCard: {
width: '30%',
backgroundColor: colors.surfaceCard,
borderRadius: 14,
borderWidth: 1,
borderColor: colors.borderDefault,
padding: 12,
alignItems: 'center',
gap: 4,
},
modeCardActive: {
backgroundColor: colors.accentPrimary,
borderColor: colors.accentPrimary,
},
modeIcon: {
fontSize: 24,
},
modeLabel: {
color: colors.textSecondary,
fontSize: 12,
fontWeight: '700',
},
modeLabelActive: {
color: colors.textPrimary,
},
cardTitle: {
color: colors.textPrimary,
fontSize: 16,
@ -181,5 +356,6 @@ const styles = StyleSheet.create({
cardBody: {
color: colors.textSecondary,
fontSize: 14,
lineHeight: 20,
},
});

19
pnpm-lock.yaml generated
View File

@ -132,6 +132,9 @@ importers:
expo:
specifier: ~55.0.4
version: 55.0.8(@babel/core@7.29.0)(@expo/dom-webview@55.0.3)(expo-router@6.0.23)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@react-native/metro-config@0.84.1(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)
expo-clipboard:
specifier: ^55.0.11
version: 55.0.11(expo@55.0.8)(react-native@0.83.2(@babel/core@7.29.0)(@react-native/metro-config@0.84.1(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
expo-constants:
specifier: ~18.0.13
version: 18.0.13(expo@55.0.8)(react-native@0.83.2(@babel/core@7.29.0)(@react-native/metro-config@0.84.1(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.0))
@ -2912,6 +2915,7 @@ packages:
'@xmldom/xmldom@0.8.11':
resolution: {integrity: sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==}
engines: {node: '>=10.0.0'}
deprecated: this version has critical issues, please update to the latest version
abort-controller@3.0.0:
resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
@ -3764,6 +3768,13 @@ packages:
react: '*'
react-native: '*'
expo-clipboard@55.0.11:
resolution: {integrity: sha512-l2zbhVdHamtK4U34zY/NpF0dd1vMcJnxtZz2CjcOudhyB9dlpuAcZMkgbELs9YTbnKWPF8+wRPKosDu8RPCUIw==}
peerDependencies:
expo: '*'
react: '*'
react-native: '*'
expo-constants@18.0.13:
resolution: {integrity: sha512-FnZn12E1dRYKDHlAdIyNFhBurKTS3F9CrfrBDJI5m3D7U17KBHMQ6JEfYlSj7LG7t+Ulr+IKaj58L1k5gBwTcQ==}
peerDependencies:
@ -8502,7 +8513,9 @@ snapshots:
metro-runtime: 0.83.5
transitivePeerDependencies:
- '@babel/core'
- bufferutil
- supports-color
- utf-8-validate
'@react-native/normalize-colors@0.83.2': {}
@ -10410,6 +10423,12 @@ snapshots:
- supports-color
- typescript
expo-clipboard@55.0.11(expo@55.0.8)(react-native@0.83.2(@babel/core@7.29.0)(@react-native/metro-config@0.84.1(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.0))(react@19.2.0):
dependencies:
expo: 55.0.8(@babel/core@7.29.0)(@expo/dom-webview@55.0.3)(expo-router@6.0.23)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@react-native/metro-config@0.84.1(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)
react: 19.2.0
react-native: 0.83.2(@babel/core@7.29.0)(@react-native/metro-config@0.84.1(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.0)
expo-constants@18.0.13(expo@55.0.8)(react-native@0.83.2(@babel/core@7.29.0)(@react-native/metro-config@0.84.1(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.0)):
dependencies:
'@expo/config': 12.0.13

View File

@ -6,7 +6,7 @@ import StarterKit from "@tiptap/starter-kit";
import Placeholder from "@tiptap/extension-placeholder";
import type { NoteDetail } from "@/lib/types";
import { useDebounce } from "@/lib/use-debounce";
import { copilotTransform, type CopilotAction } from "@/lib/copilot-client";
import { copilotTransform, type CopilotAction, type CopilotTone } from "@/lib/copilot-client";
import { toast } from "@/lib/toast";
const TOOLBAR_BTN: React.CSSProperties = {
@ -53,6 +53,8 @@ export function NoteEditor({
const [title, setTitle] = useState(note.title);
const [, setBodyTick] = useState(0);
const [copilotBusy, setCopilotBusy] = useState(false);
const [toneMenuOpen, setToneMenuOpen] = useState(false);
const [explainResult, setExplainResult] = useState<string | null>(null);
const onSaveRef = useRef(onSave);
onSaveRef.current = onSave;
@ -102,17 +104,45 @@ export function NoteEditor({
);
const runCopilot = useCallback(
async (action: CopilotAction) => {
async (action: CopilotAction, tone?: CopilotTone) => {
if (!editor || !copilotNoteId || !copilotWorkspaceId) return;
const { from, to } = editor.state.selection;
const selected = editor.state.doc.textBetween(from, to, "\n").trim();
// "continue" uses all text before cursor, not selection
if (action === "continue") {
const fullText = editor.state.doc.textBetween(0, editor.state.selection.to, "\n").trim();
if (!fullText) { toast.error("Place cursor in the editor first"); return; }
setCopilotBusy(true);
try {
const out = await copilotTransform(copilotNoteId, copilotWorkspaceId, action, fullText);
const escaped = out.split("\n").map((l) => l.replace(/</g, "&lt;").replace(/>/g, "&gt;")).join("</p><p>");
editor.chain().focus().insertContent(`<p>${escaped}</p>`).run();
toast.success("Continuation inserted — review and save");
} catch (e) { toast.error(e instanceof Error ? e.message : "Continue failed"); }
finally { setCopilotBusy(false); }
return;
}
// "explain" shows result in a tooltip, doesn't replace text
if (action === "explain") {
if (!selected) { toast.error("Select text to explain"); return; }
setCopilotBusy(true);
try {
const out = await copilotTransform(copilotNoteId, copilotWorkspaceId, action, selected);
setExplainResult(out);
} catch (e) { toast.error(e instanceof Error ? e.message : "Explain failed"); }
finally { setCopilotBusy(false); }
return;
}
if (!selected) {
toast.error("Select text in the editor first");
return;
}
setCopilotBusy(true);
try {
const out = await copilotTransform(copilotNoteId, copilotWorkspaceId, action, selected);
const out = await copilotTransform(copilotNoteId, copilotWorkspaceId, action, selected, tone);
const escaped = out
.split("\n")
.map((line) => line.replace(/</g, "&lt;").replace(/>/g, "&gt;"))
@ -181,9 +211,31 @@ export function NoteEditor({
{a}
</button>
))}
<span style={{ width: 1, background: "var(--nl-border-default)", margin: "0 4px", alignSelf: "stretch" }} />
<span style={{ color: "var(--nl-text-secondary)", fontSize: "var(--nl-fs-sm)", marginRight: 4 }}>AI</span>
<button type="button" disabled={copilotBusy} style={{ ...TOOLBAR_BTN, opacity: copilotBusy ? 0.5 : 1 }} onClick={() => void runCopilot("fix-rewrite")}>Fix & Rewrite</button>
<div style={{ position: "relative" }}>
<button type="button" disabled={copilotBusy} style={{ ...TOOLBAR_BTN, opacity: copilotBusy ? 0.5 : 1 }} onClick={() => setToneMenuOpen(!toneMenuOpen)}>Change Tone </button>
{toneMenuOpen && (
<div style={{ position: "absolute", top: "100%", left: 0, zIndex: 50, background: "var(--nl-surface-card)", border: "1px solid var(--nl-border-default)", borderRadius: "var(--nl-radius-md)", padding: 4, display: "grid", gap: 2, minWidth: 120 }}>
{(["formal", "casual", "professional", "friendly"] as const).map((t) => (
<button key={t} type="button" style={{ ...TOOLBAR_BTN, textAlign: "left", width: "100%" }} onClick={() => { setToneMenuOpen(false); void runCopilot("change-tone", t); }}>{t}</button>
))}
</div>
)}
</div>
<button type="button" disabled={copilotBusy} style={{ ...TOOLBAR_BTN, opacity: copilotBusy ? 0.5 : 1 }} onClick={() => void runCopilot("continue")}>Continue </button>
<button type="button" disabled={copilotBusy} style={{ ...TOOLBAR_BTN, opacity: copilotBusy ? 0.5 : 1 }} onClick={() => void runCopilot("explain")}>Explain</button>
</div>
) : null}
{explainResult && (
<div style={{ background: "var(--nl-surface-card)", border: "1px solid var(--nl-border-default)", borderRadius: "var(--nl-radius-md)", padding: "var(--nl-space-3)", fontSize: "var(--nl-fs-sm)", color: "var(--nl-text-secondary)", display: "flex", gap: 8, alignItems: "flex-start" }}>
<div style={{ flex: 1 }}>{explainResult}</div>
<button type="button" style={{ ...TOOLBAR_BTN, fontSize: 12 }} onClick={() => setExplainResult(null)}></button>
</div>
)}
<EditorContent editor={editor} />
</div>

View File

@ -2,18 +2,20 @@
import { createNotesApiClient } from "@/lib/api-helpers";
export type CopilotAction = "shorten" | "expand" | "bulletize" | "grammar";
export type CopilotAction = "shorten" | "expand" | "bulletize" | "grammar" | "fix-rewrite" | "change-tone" | "continue" | "explain";
export type CopilotTone = "formal" | "casual" | "professional" | "friendly";
export async function copilotTransform(
noteId: string,
workspaceId: string,
action: CopilotAction,
text: string,
tone?: CopilotTone,
): Promise<string> {
const api = createNotesApiClient();
const res = await api.fetch<{ text: string }>(`/notes/${encodeURIComponent(noteId)}/copilot`, {
method: "POST",
body: JSON.stringify({ workspaceId, action, text }),
body: JSON.stringify({ workspaceId, action, text, ...(tone ? { tone } : {}) }),
});
return res.text;
}