fix(tracker-seed): cap dedupe list at limit=100 + auto-register products
Two bugs caused duplicate items on re-run: the dedupe list used limit=500
(server caps at 100 -> 400 -> silent empty set -> dupes), and meta productIds
weren't registered so GET /items 400'd ("Unknown product"). Now registers every
referenced product first (idempotent) and lists with limit=100; dedupe failures
are logged loudly. Verified idempotent: re-run skips all 16.
Generated with [Devin](https://cli.devin.ai/docs)
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
ae7909018a
commit
abc8a0f517
@ -63,7 +63,8 @@ function mintToken() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function listTitles(token, productId) {
|
async function listTitles(token, productId) {
|
||||||
const url = `${API}/api/items?productId=${encodeURIComponent(productId)}&limit=500`;
|
// limit max is 100 (server caps it); 500 would 400 and silently break dedupe.
|
||||||
|
const url = `${API}/api/items?productId=${encodeURIComponent(productId)}&limit=100`;
|
||||||
const res = await fetch(url, {
|
const res = await fetch(url, {
|
||||||
headers: { authorization: `Bearer ${token}`, 'x-product-id': productId },
|
headers: { authorization: `Bearer ${token}`, 'x-product-id': productId },
|
||||||
});
|
});
|
||||||
@ -72,6 +73,36 @@ async function listTitles(token, productId) {
|
|||||||
return new Set((data.items || []).map((i) => i.title));
|
return new Set((data.items || []).map((i) => i.title));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Derive an uppercase-letters-only license prefix (<=8) from a productId. */
|
||||||
|
function licensePrefix(productId) {
|
||||||
|
return productId.replace(/[^a-z]/gi, '').toUpperCase().slice(0, 8) || 'PRD';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure a product exists before creating items under it. POST /items skips
|
||||||
|
* product validation when productId is in the body, but GET /items validates —
|
||||||
|
* so without registration the dedupe list 400s ("Unknown product") and
|
||||||
|
* re-runs create duplicates. Idempotent: treats 409/200/201 as success.
|
||||||
|
*/
|
||||||
|
async function ensureProduct(token, productId) {
|
||||||
|
const res = await fetch(`${API}/api/products`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
authorization: `Bearer ${token}`,
|
||||||
|
'content-type': 'application/json',
|
||||||
|
'x-product-id': productId,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
productId,
|
||||||
|
displayName: productId.charAt(0).toUpperCase() + productId.slice(1),
|
||||||
|
licensePrefix: licensePrefix(productId),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
if (!res.ok && res.status !== 409) {
|
||||||
|
console.warn(` warn: register ${productId} -> HTTP ${res.status}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function createItem(token, item) {
|
async function createItem(token, item) {
|
||||||
const res = await fetch(`${API}/api/items`, {
|
const res = await fetch(`${API}/api/items`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
@ -102,6 +133,13 @@ async function main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const token = mintToken();
|
const token = mintToken();
|
||||||
|
|
||||||
|
// Register every referenced product first so the dedupe list (GET /items)
|
||||||
|
// doesn't 400 on unknown products — which would silently cause duplicates.
|
||||||
|
const productIds = [...new Set(items.map((it) => it.productId))];
|
||||||
|
console.log(`Ensuring ${productIds.length} product(s) are registered...`);
|
||||||
|
for (const pid of productIds) await ensureProduct(token, pid);
|
||||||
|
|
||||||
const titleCache = new Map();
|
const titleCache = new Map();
|
||||||
let created = 0;
|
let created = 0;
|
||||||
let skipped = 0;
|
let skipped = 0;
|
||||||
@ -111,7 +149,13 @@ async function main() {
|
|||||||
try {
|
try {
|
||||||
if (!FORCE) {
|
if (!FORCE) {
|
||||||
if (!titleCache.has(it.productId)) {
|
if (!titleCache.has(it.productId)) {
|
||||||
titleCache.set(it.productId, await listTitles(token, it.productId).catch(() => new Set()));
|
titleCache.set(
|
||||||
|
it.productId,
|
||||||
|
await listTitles(token, it.productId).catch((e) => {
|
||||||
|
console.warn(` warn: dedupe list for ${it.productId} failed (${e.message}); may create duplicates`);
|
||||||
|
return new Set();
|
||||||
|
})
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if (titleCache.get(it.productId).has(it.title)) {
|
if (titleCache.get(it.productId).has(it.title)) {
|
||||||
console.log(` SKIP ${it.productId.padEnd(16)} ${it.title} (already exists)`);
|
console.log(` SKIP ${it.productId.padEnd(16)} ${it.title} (already exists)`);
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user