initial push from external dev branches

This commit is contained in:
iFlip721
2026-06-11 16:40:21 -04:00
parent 986e83632b
commit 245034a43a
182 changed files with 15465 additions and 0 deletions

39
server/dist/sources/core/buildSource.js vendored Normal file
View File

@@ -0,0 +1,39 @@
// The generic "standard function" pipeline, run once per source for a LIVE sync:
// listChannels() → raw upstream records → normalize(raw) → one SourceChannel doc each
// → dedupe by deterministic _id (last wins, idempotent) → return docs for upsert.
//
// Ported from d-combine/lib/core/build-source.mjs, but returns the docs instead of writing files —
// the seed module upserts them into Mongo. The whole file is source-agnostic.
import { logger } from './logger.js';
export async function buildSource(adapter) {
const startedAt = Date.now();
logger.info('build', `[${adapter.id}] fetching channel listings…`);
const { raw, meta = {} } = await adapter.listChannels();
logger.info('build', `[${adapter.id}] got ${raw.length} raw records (${meta.live === false ? 'offline snapshot' : 'live'})`);
const ingestedAt = new Date().toISOString();
const normalized = [];
for (const record of raw) {
try {
const doc = adapter.normalize(record, { ingestedAt });
if (doc)
normalized.push(doc);
}
catch (err) {
logger.warn('build', `[${adapter.id}] skipped a record: ${err.message}`);
}
}
// Dedupe by deterministic _id (last wins) — guards against duplicate upstream rows.
const byId = new Map();
for (const doc of normalized)
byId.set(doc._id, doc);
const docs = [...byId.values()];
logger.ok('build', `[${adapter.id}] normalized ${docs.length} docs`);
return {
id: adapter.id,
count: docs.length,
docs,
live: meta.live !== false,
meta: { ...meta, buildMs: Date.now() - startedAt },
};
}
//# sourceMappingURL=buildSource.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"buildSource.js","sourceRoot":"","sources":["../../../src/sources/core/buildSource.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,yFAAyF;AACzF,oFAAoF;AACpF,EAAE;AACF,mGAAmG;AACnG,8EAA8E;AAE9E,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAYrC,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAsB;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,8BAA8B,CAAC,CAAC;IAEnE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,EAAE,EAAE,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IACxD,MAAM,CAAC,IAAI,CACT,OAAO,EACP,IAAI,OAAO,CAAC,EAAE,SAAS,GAAG,CAAC,MAAM,iBAAiB,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,GAAG,CACvG,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAuB,EAAE,CAAC;IAC1C,KAAK,MAAM,MAAM,IAAI,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;YACtD,IAAI,GAAG;gBAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,uBAAwB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAED,oFAAoF;IACpF,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4B,CAAC;IACjD,KAAK,MAAM,GAAG,IAAI,UAAU;QAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAEhC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,gBAAgB,IAAI,CAAC,MAAM,OAAO,CAAC,CAAC;IACrE,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,IAAI;QACJ,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,KAAK;QACzB,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE;KACnD,CAAC;AACJ,CAAC"}

18
server/dist/sources/core/logger.js vendored Normal file
View File

@@ -0,0 +1,18 @@
// Tiny tagged logger matching the existing console style ("[mongo] connected", "[api] …").
// Ported from d-combine/lib/core/logger.mjs, trimmed to what the core uses.
function emit(level, tag, msg) {
const line = `[${tag}] ${msg}`;
if (level === 'error')
console.error(line);
else if (level === 'warn')
console.warn(line);
else
console.info(line);
}
export const logger = {
info: (tag, msg) => emit('info', tag, msg),
warn: (tag, msg) => emit('warn', tag, msg),
error: (tag, msg) => emit('error', tag, msg),
ok: (tag, msg) => emit('ok', tag, msg),
};
//# sourceMappingURL=logger.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../../src/sources/core/logger.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,4EAA4E;AAI5E,SAAS,IAAI,CAAC,KAAY,EAAE,GAAW,EAAE,GAAW;IAClD,MAAM,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;IAC/B,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACtC,IAAI,KAAK,KAAK,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;QACzC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,IAAI,EAAE,CAAC,GAAW,EAAE,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IAC1D,IAAI,EAAE,CAAC,GAAW,EAAE,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IAC1D,KAAK,EAAE,CAAC,GAAW,EAAE,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC;IAC5D,EAAE,EAAE,CAAC,GAAW,EAAE,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC;CACvD,CAAC"}

35
server/dist/sources/core/metrics.js vendored Normal file
View File

@@ -0,0 +1,35 @@
// In-memory counters, one set PER SOURCE. Surfaced via /api/sources/:id/status and /health-style
// reporting. Ported from d-combine/lib/core/metrics.mjs.
export function createMetrics() {
return {
startedAt: Date.now(),
requests: { total: 0, master: 0, variant: 0, segment: 0, other: 0, errors: 0 },
upstream: { ok: 0, notLive: 0, forbidden: 0, failed: 0 }, // 404=notLive, 403=forbidden(gate)
bytesStreamed: 0,
active: 0,
lastStreamAt: null,
lastError: null,
};
}
/** Human-readable byte size. */
export function fmtBytes(n) {
if (n < 1024)
return `${n}B`;
if (n < 1048576)
return `${(n / 1024).toFixed(1)}KB`;
return `${(n / 1048576).toFixed(2)}MB`;
}
/** JSON snapshot of one source's metrics. */
export function snapshotOne(m) {
return {
uptimeSeconds: Math.round((Date.now() - m.startedAt) / 1000),
active: m.active,
requests: { ...m.requests },
upstream: { ...m.upstream },
bytesStreamed: m.bytesStreamed,
mbStreamed: +(m.bytesStreamed / 1048576).toFixed(2),
lastStreamAt: m.lastStreamAt ? new Date(m.lastStreamAt).toISOString() : null,
lastError: m.lastError,
};
}
//# sourceMappingURL=metrics.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/sources/core/metrics.ts"],"names":[],"mappings":"AAAA,iGAAiG;AACjG,yDAAyD;AAmBzD,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC9E,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,mCAAmC;QAC7F,aAAa,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC;QACT,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,gCAAgC;AAChC,MAAM,UAAU,QAAQ,CAAC,CAAS;IAChC,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC;IAC7B,IAAI,CAAC,GAAG,OAAO;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrD,OAAO,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,WAAW,CAAC,CAAU;IACpC,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;QAC5D,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE;QAC3B,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE;QAC3B,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACnD,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;QAC5E,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC;AACJ,CAAC"}

43
server/dist/sources/core/playlist.js vendored Normal file
View File

@@ -0,0 +1,43 @@
// Shared HLS playlist helpers. Ported from d-combine/lib/core/playlist.mjs. The only per-source
// difference — whether the rewriter also learns (allowlists) each child host — is the `onChildHost`
// hook param, so one implementation serves every source.
/** True if the upstream URL / content-type looks like an HLS playlist (.m3u8). */
export function looksLikePlaylist(upstreamUrl, contentType) {
if (contentType && contentType.includes('mpegurl'))
return true; // apple.mpegurl / x-mpegurl
try {
return new URL(upstreamUrl).pathname.toLowerCase().endsWith('.m3u8');
}
catch {
return false;
}
}
/**
* Rewrite every child URI in a playlist so it routes back through this proxy.
*
* @param text the raw playlist body
* @param baseUrl the upstream URL it was fetched from (for relative→absolute)
* @param prefix proxy mount prefix to prepend, e.g. "/api/v1/dulo/"
* @param onChildHost per-child-host hook (dlhd dynamic-allow; dulo/common null)
*/
export function rewritePlaylist(text, baseUrl, prefix, onChildHost) {
return text
.split(/\r?\n/)
.map((rawLine) => {
const trimmed = rawLine.trim();
if (!trimmed || trimmed.startsWith('#'))
return rawLine; // tag / comment / blank → as-is
const abs = new URL(trimmed, baseUrl).href; // resolve relative → absolute
if (onChildHost) {
try {
onChildHost(new URL(abs).hostname);
}
catch {
/* ignore malformed */
}
}
return `${prefix}${encodeURIComponent(abs)}`;
})
.join('\n');
}
//# sourceMappingURL=playlist.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"playlist.js","sourceRoot":"","sources":["../../../src/sources/core/playlist.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,oGAAoG;AACpG,yDAAyD;AAEzD,kFAAkF;AAClF,MAAM,UAAU,iBAAiB,CAAC,WAAmB,EAAE,WAAmB;IACxE,IAAI,WAAW,IAAI,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,4BAA4B;IAC7F,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,OAAe,EACf,MAAc,EACd,WAA4C;IAE5C,OAAO,IAAI;SACR,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACf,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC,CAAC,gCAAgC;QACzF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,8BAA8B;QAC1E,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YACrC,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;IAC/C,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}

154
server/dist/sources/core/proxyHandler.js vendored Normal file
View File

@@ -0,0 +1,154 @@
// One Express handler factory per source, bound to GET /api/v1/:source/*. Ported from
// d-combine/lib/core/proxy-handler.mjs. Every per-source difference (resolve, headers, SSRF allow,
// dynamic-allow, segment relabel, artifact classification) is read off `adapter`; the control flow
// below is invariant.
//
// /api/v1/<source>/<enc entry-or-stream URL>
// · entry URL (dlhd watch.php / stream-N.php) → adapter.resolveStream() → fresh master, then proxy
// · master/variant .m3u8 → rewrite child URIs back through /api/v1/<source>/…
// · segment → pipe bytes (adapter may relabel the content-type)
import { Readable } from 'node:stream';
import { logger } from './logger.js';
import { fmtBytes } from './metrics.js';
import { looksLikePlaylist, rewritePlaylist } from './playlist.js';
function label(url) {
try {
const u = new URL(url);
const file = u.pathname.split('/').pop() || '';
return { host: u.hostname, short: file.slice(0, 8) || '/' };
}
catch {
return { host: '?', short: '?' };
}
}
export function createProxyHandler(adapter, metrics) {
// Marker used to slice the raw (still-encoded) upstream URL out of req.originalUrl, independent of
// where the router is mounted. Keeps embedded ?session=/?md5&expires through ONE decodeURIComponent.
const MARKER = `/v1/${adapter.id}/`;
const PREFIX = `/api/v1/${adapter.id}/`;
const tag = `stream:${adapter.id}`;
const { proxy } = adapter;
return async function handler(req, res) {
const startedAt = Date.now();
const ms = () => `${Date.now() - startedAt}ms`;
// 1. Extract + decode the single percent-encoded upstream URL segment.
const idx = req.originalUrl.indexOf(MARKER);
const rawPath = idx >= 0 ? req.originalUrl.slice(idx + MARKER.length) : '';
let upstreamUrl;
try {
upstreamUrl = decodeURIComponent(rawPath);
}
catch {
logger.warn(tag, `400 malformed encoded URL from ${req.ip}`);
res.status(400).type('text/plain').send('Bad request: malformed encoded URL');
return;
}
// 2. Resolve-then-proxy: an entry URL must become a fresh stream URL first
// (dulo/common: never — entry IS the master; dlhd: the 3-hop scrape).
if (adapter.isEntryUrl(upstreamUrl)) {
metrics.requests.total++;
metrics.requests.master++;
try {
const resolved = await adapter.resolveStream(upstreamUrl);
upstreamUrl = resolved.masterUrl;
}
catch (err) {
metrics.requests.errors++;
metrics.upstream.notLive++;
metrics.lastError = err.message;
logger.warn(tag, `resolve failed: ${err.message} (${ms()})`);
res.status(502).type('text/plain').send(`Resolve failed: ${err.message}`);
return;
}
}
else {
// 3. Direct hop (master/variant/segment from a rewritten playlist) → SSRF gate.
if (!proxy.isAllowedUpstream(upstreamUrl)) {
logger.warn(tag, `400 blocked upstream: ${String(upstreamUrl).slice(0, 80)}`);
res.status(400).type('text/plain').send('Bad request: upstream host not in the allowlist');
return;
}
metrics.requests.total++;
metrics.requests[proxy.classifyArtifact(upstreamUrl)]++;
}
const type = proxy.classifyArtifact(upstreamUrl);
const { host, short } = label(upstreamUrl);
metrics.active++;
res.on('close', () => {
metrics.active--;
});
let upstream;
try {
upstream = await fetch(upstreamUrl, { headers: proxy.upstreamHeaders(upstreamUrl) });
}
catch (err) {
metrics.upstream.failed++;
metrics.requests.errors++;
metrics.lastError = err.message;
logger.error(tag, `${type} ${host} ${short} upstream fetch failed: ${err.message} (${ms()})`);
res.status(502).type('text/plain').send(`Upstream fetch failed: ${err.message}`);
return;
}
// Forward upstream errors verbatim. 404 = not transcoding right now; 403 = origin/referer gate.
if (!upstream.ok) {
metrics.requests.errors++;
if (upstream.status === 404)
metrics.upstream.notLive++;
else if (upstream.status === 403)
metrics.upstream.forbidden++;
else
metrics.upstream.failed++;
const note = upstream.status === 404 ? ' (not live)' : upstream.status === 403 ? ' (gate)' : '';
metrics.lastError = `HTTP ${upstream.status} ${host}/${short}`;
logger.warn(tag, `${type} ${host} ${short} status=${upstream.status}${note} (${ms()})`);
const detail = await upstream.text().catch(() => '');
res
.status(upstream.status)
.type(upstream.headers.get('content-type') || 'text/plain')
.send(detail || `Upstream HTTP ${upstream.status}`);
return;
}
const contentType = upstream.headers.get('content-type') || '';
// 4. Playlist → rewrite child URIs back through this source's proxy prefix
// (and let the adapter learn each child host: dlhd dynamic-allow; dulo/common no-op).
if (looksLikePlaylist(upstreamUrl, contentType)) {
const rewritten = rewritePlaylist(await upstream.text(), upstreamUrl, PREFIX, proxy.onPlaylistChildHost);
const bytes = Buffer.byteLength(rewritten);
metrics.upstream.ok++;
metrics.bytesStreamed += bytes;
metrics.lastStreamAt = Date.now();
logger.ok(tag, `${type} ${host} ${short} status=200 ${fmtBytes(bytes)} (${ms()})`);
res.set('Cache-Control', 'no-store'); // playlists + tokens are short-lived
res.type('application/vnd.apple.mpegurl').send(rewritten);
return;
}
// 5. Segment (or anything else) → stream the bytes through, content-type per the adapter.
res.set('Content-Type', proxy.relabelSegmentContentType(upstreamUrl, contentType, type));
res.set('Cache-Control', 'no-store');
if (!upstream.body) {
metrics.upstream.ok++;
logger.ok(tag, `${type} ${host} ${short} status=200 0B (${ms()})`);
res.end();
return;
}
let bytes = 0;
const body = Readable.fromWeb(upstream.body);
body.on('data', (chunk) => {
bytes += chunk.length;
});
res.on('finish', () => {
metrics.upstream.ok++;
metrics.bytesStreamed += bytes;
metrics.lastStreamAt = Date.now();
logger.ok(tag, `${type} ${host} ${short} status=200 ${fmtBytes(bytes)} (${ms()})`);
});
body.on('error', (err) => {
metrics.upstream.failed++;
metrics.lastError = err.message;
logger.error(tag, `${type} ${host} ${short} stream error: ${err.message}`);
res.destroy(err);
});
body.pipe(res);
};
}
//# sourceMappingURL=proxyHandler.js.map

File diff suppressed because one or more lines are too long