Hermy HQ: self-hostable mission-control template for your Hermes agent
A Next.js 16 + Prisma/Postgres dashboard that pairs with a local Hermes agent over a Postgres message bus: dispatch work, approve side-effecting actions, browse the agent's memory, and watch it run. Ships with an agent-onboarding prompt (ONBOARDING.md) so your Hermes can install it for you step by step. All secrets are env-configured; nothing sensitive is bundled. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
sharbelxyz
Claude Opus 4.8
@@ -0,0 +1,325 @@
|
||||
const https = require('https');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// ── Config ──────────────────────────────────────────────────────────────────────
|
||||
const NICHES = {
|
||||
'openclaw': { label: '🦞 OpenClaw & AI Agents', queries: ['OpenClaw tutorial', 'build AI agents OpenClaw', 'run AI agents locally 2026'] },
|
||||
'polymarket-trading': { label: '📈 Polymarket & Trading Bots', queries: ['Polymarket trading bot', 'prediction market bot', 'AI trading bot build 2026'] },
|
||||
'ai-agent-build': { label: '🤖 AI Agent Builds', queries: ['build AI agent no code', 'AI automation agent tutorial', 'how to build AI employees'] },
|
||||
'agency-growth': { label: '🏢 Agency & Clients', queries: ['marketing agency growth', 'how to grow a marketing agency', 'agency owner content creator'] },
|
||||
'ai-tools': { label: '🛠️ AI for Creators', queries: ['AI tools content creators', 'AI automation marketing', 'AI replace marketing team'] },
|
||||
};
|
||||
|
||||
const MAX_CHANNEL_SUBS = 1000000;
|
||||
const MIN_OUTLIER_SCORE = 1.5;
|
||||
const DAYS_AGO = 30;
|
||||
const MAX_RESULTS = 15;
|
||||
|
||||
// ── Load API key from .env.local ────────────────────────────────────────────────
|
||||
function loadApiKey() {
|
||||
const envPath = path.join(__dirname, '..', '.env.local');
|
||||
if (!fs.existsSync(envPath)) {
|
||||
console.error('ERROR: .env.local not found at', envPath);
|
||||
process.exit(1);
|
||||
}
|
||||
const lines = fs.readFileSync(envPath, 'utf-8').split('\n');
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith('YOUTUBE_API_KEY=')) {
|
||||
const val = trimmed.slice('YOUTUBE_API_KEY='.length).trim();
|
||||
return val.replace(/^["']|["']$/g, '');
|
||||
}
|
||||
}
|
||||
console.error('ERROR: YOUTUBE_API_KEY not found in .env.local');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Add one or more YouTube Data API keys here (or via YOUTUBE_API_KEY in .env.local).
|
||||
// Multiple keys are rotated when quota is hit.
|
||||
const API_KEYS = [loadApiKey()].filter(Boolean);
|
||||
let currentKeyIndex = 0;
|
||||
function getApiKey() { return API_KEYS[currentKeyIndex]; }
|
||||
function rotateKey() {
|
||||
currentKeyIndex = (currentKeyIndex + 1) % API_KEYS.length;
|
||||
console.log(` ⚡ Rotated to API key ${currentKeyIndex + 1}`);
|
||||
}
|
||||
|
||||
// ── HTTP helper ─────────────────────────────────────────────────────────────────
|
||||
function fetchJSON(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, (res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => { data += chunk; });
|
||||
res.on('end', () => {
|
||||
try { resolve(JSON.parse(data)); }
|
||||
catch (e) { reject(new Error('JSON parse error: ' + data.slice(0, 200))); }
|
||||
});
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
// ── ISO 8601 duration parser ────────────────────────────────────────────────────
|
||||
function parseDuration(iso) {
|
||||
if (!iso) return 0;
|
||||
const match = iso.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
|
||||
if (!match) return 0;
|
||||
const h = parseInt(match[1] || '0', 10);
|
||||
const m = parseInt(match[2] || '0', 10);
|
||||
const s = parseInt(match[3] || '0', 10);
|
||||
return h * 3600 + m * 60 + s;
|
||||
}
|
||||
|
||||
function formatDuration(seconds) {
|
||||
const h = Math.floor(seconds / 3600);
|
||||
const m = Math.floor((seconds % 3600) / 60);
|
||||
const s = seconds % 60;
|
||||
if (h > 0) return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
|
||||
return `${m}:${String(s).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function median(arr) {
|
||||
if (arr.length === 0) return 0;
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 !== 0 ? sorted[mid] : Math.floor((sorted[mid - 1] + sorted[mid]) / 2);
|
||||
}
|
||||
|
||||
// ── YouTube API calls ───────────────────────────────────────────────────────────
|
||||
async function searchVideos(query, publishedAfter) {
|
||||
const url = `https://www.googleapis.com/youtube/v3/search?part=snippet&q=${encodeURIComponent(query)}&type=video&order=relevance&publishedAfter=${publishedAfter}&maxResults=${MAX_RESULTS}&key=${getApiKey()}`;
|
||||
const data = await fetchJSON(url);
|
||||
if (data.error) {
|
||||
if (data.error.message && data.error.message.includes('quota')) {
|
||||
rotateKey();
|
||||
const url2 = `https://www.googleapis.com/youtube/v3/search?part=snippet&q=${encodeURIComponent(query)}&type=video&order=relevance&publishedAfter=${publishedAfter}&maxResults=${MAX_RESULTS}&key=${getApiKey()}`;
|
||||
const data2 = await fetchJSON(url2);
|
||||
if (data2.error) { console.error(' Search API error (both keys exhausted):', data2.error.message); return []; }
|
||||
return (data2.items || []).map((item) => ({ id: item.id.videoId, title: item.snippet.title, publishedAt: item.snippet.publishedAt, channelId: item.snippet.channelId, channelTitle: item.snippet.channelTitle, thumbnail: item.snippet.thumbnails.high ? item.snippet.thumbnails.high.url : item.snippet.thumbnails.default.url }));
|
||||
}
|
||||
console.error(' Search API error:', data.error.message);
|
||||
return [];
|
||||
}
|
||||
return (data.items || []).map((item) => ({
|
||||
id: item.id.videoId,
|
||||
title: item.snippet.title,
|
||||
publishedAt: item.snippet.publishedAt,
|
||||
channelId: item.snippet.channelId,
|
||||
channelTitle: item.snippet.channelTitle,
|
||||
thumbnail: item.snippet.thumbnails.high ? item.snippet.thumbnails.high.url : item.snippet.thumbnails.default.url,
|
||||
}));
|
||||
}
|
||||
|
||||
async function getVideoDetails(videoIds) {
|
||||
if (videoIds.length === 0) return {};
|
||||
const url = `https://www.googleapis.com/youtube/v3/videos?part=statistics,contentDetails&id=${videoIds.join(',')}&key=${getApiKey()}`;
|
||||
const data = await fetchJSON(url);
|
||||
if (data.error) {
|
||||
console.error(' Video details API error:', data.error.message);
|
||||
return {};
|
||||
}
|
||||
const map = {};
|
||||
for (const item of (data.items || [])) {
|
||||
const dur = parseDuration(item.contentDetails.duration);
|
||||
map[item.id] = {
|
||||
views: parseInt(item.statistics.viewCount || '0', 10),
|
||||
likes: parseInt(item.statistics.likeCount || '0', 10),
|
||||
comments: parseInt(item.statistics.commentCount || '0', 10),
|
||||
duration: formatDuration(dur),
|
||||
durationSeconds: dur,
|
||||
};
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
async function getChannelInfo(channelId) {
|
||||
const url = `https://www.googleapis.com/youtube/v3/channels?part=statistics,snippet,contentDetails&id=${channelId}&key=${getApiKey()}`;
|
||||
const data = await fetchJSON(url);
|
||||
if (data.error || !data.items || data.items.length === 0) return null;
|
||||
const ch = data.items[0];
|
||||
return {
|
||||
subs: parseInt(ch.statistics.subscriberCount || '0', 10),
|
||||
uploadsPlaylistId: ch.contentDetails.relatedPlaylists.uploads,
|
||||
thumb: ch.snippet.thumbnails.default ? ch.snippet.thumbnails.default.url : '',
|
||||
};
|
||||
}
|
||||
|
||||
async function getChannelMedianViews(uploadsPlaylistId) {
|
||||
const url = `https://www.googleapis.com/youtube/v3/playlistItems?part=contentDetails&playlistId=${uploadsPlaylistId}&maxResults=30&key=${getApiKey()}`;
|
||||
const data = await fetchJSON(url);
|
||||
if (data.error || !data.items || data.items.length === 0) return 0;
|
||||
const videoIds = data.items.map((i) => i.contentDetails.videoId);
|
||||
// Batch in groups of 50 (API limit)
|
||||
const allViews = [];
|
||||
for (let i = 0; i < videoIds.length; i += 50) {
|
||||
const batch = videoIds.slice(i, i + 50);
|
||||
const vUrl = `https://www.googleapis.com/youtube/v3/videos?part=statistics&id=${batch.join(',')}&key=${getApiKey()}`;
|
||||
const vData = await fetchJSON(vUrl);
|
||||
if (vData.items) {
|
||||
for (const v of vData.items) {
|
||||
allViews.push(parseInt(v.statistics.viewCount || '0', 10));
|
||||
}
|
||||
}
|
||||
}
|
||||
return median(allViews);
|
||||
}
|
||||
|
||||
// ── Main scanner ────────────────────────────────────────────────────────────────
|
||||
async function main() {
|
||||
console.log('=== YouTube Outlier Scanner ===');
|
||||
console.log(`Scanning ${Object.keys(NICHES).length} niches, ${Object.values(NICHES).reduce((s, n) => s + n.queries.length, 0)} queries total\n`);
|
||||
|
||||
const publishedAfter = new Date(Date.now() - DAYS_AGO * 24 * 60 * 60 * 1000).toISOString();
|
||||
const seenIds = new Set();
|
||||
const allVideos = [];
|
||||
const nicheCounts = {};
|
||||
const channelCache = {}; // channelId -> { subs, medianViews, thumb }
|
||||
|
||||
for (const [nicheKey, niche] of Object.entries(NICHES)) {
|
||||
console.log(`\n── ${niche.label} ──`);
|
||||
nicheCounts[nicheKey] = { label: niche.label, count: 0 };
|
||||
|
||||
for (const query of niche.queries) {
|
||||
console.log(` Searching: "${query}"`);
|
||||
|
||||
const results = await searchVideos(query, publishedAfter);
|
||||
if (results.length === 0) { console.log(' No results'); continue; }
|
||||
|
||||
// Filter already-seen
|
||||
const fresh = results.filter((v) => !seenIds.has(v.id));
|
||||
if (fresh.length === 0) { console.log(' All duplicates, skipping'); continue; }
|
||||
|
||||
// Get video details
|
||||
const details = await getVideoDetails(fresh.map((v) => v.id));
|
||||
|
||||
for (const video of fresh) {
|
||||
if (seenIds.has(video.id)) continue;
|
||||
const det = details[video.id];
|
||||
if (!det) continue;
|
||||
|
||||
// Skip Shorts (<=90s)
|
||||
if (det.durationSeconds <= 90) continue;
|
||||
|
||||
// Get/cache channel info
|
||||
if (!channelCache[video.channelId]) {
|
||||
const chInfo = await getChannelInfo(video.channelId);
|
||||
if (!chInfo) continue;
|
||||
if (chInfo.subs > MAX_CHANNEL_SUBS) {
|
||||
channelCache[video.channelId] = { subs: chInfo.subs, skip: true };
|
||||
continue;
|
||||
}
|
||||
const medViews = await getChannelMedianViews(chInfo.uploadsPlaylistId);
|
||||
channelCache[video.channelId] = { subs: chInfo.subs, medianViews: medViews, thumb: chInfo.thumb, skip: false };
|
||||
}
|
||||
|
||||
const ch = channelCache[video.channelId];
|
||||
if (ch.skip) continue;
|
||||
if (ch.medianViews === 0) continue;
|
||||
|
||||
const outlierScore = Math.round((det.views / ch.medianViews) * 10) / 10;
|
||||
if (outlierScore < MIN_OUTLIER_SCORE) continue;
|
||||
|
||||
const daysSincePublish = Math.max(1, Math.floor((Date.now() - new Date(video.publishedAt).getTime()) / (1000 * 60 * 60 * 24)));
|
||||
const velocityScore = Math.round(det.views / daysSincePublish); // views per day
|
||||
|
||||
seenIds.add(video.id);
|
||||
nicheCounts[nicheKey].count++;
|
||||
|
||||
allVideos.push({
|
||||
id: video.id,
|
||||
title: video.title,
|
||||
publishedAt: video.publishedAt,
|
||||
thumbnail: video.thumbnail,
|
||||
views: det.views,
|
||||
likes: det.likes,
|
||||
comments: det.comments,
|
||||
duration: det.duration,
|
||||
durationSeconds: det.durationSeconds,
|
||||
outlierScore,
|
||||
velocityScore,
|
||||
daysSincePublish,
|
||||
niche: nicheKey,
|
||||
nicheLabel: niche.label,
|
||||
query,
|
||||
channelId: video.channelId,
|
||||
channelTitle: video.channelTitle,
|
||||
channelSubs: ch.subs,
|
||||
channelMedianViews: ch.medianViews,
|
||||
channelThumb: ch.thumb,
|
||||
});
|
||||
|
||||
console.log(` ✓ ${outlierScore}x | ${det.views.toLocaleString()} views | ${video.title.slice(0, 60)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by outlierScore descending
|
||||
allVideos.sort((a, b) => b.outlierScore - a.outlierScore);
|
||||
|
||||
const now = new Date();
|
||||
const dateStr = now.toISOString().slice(0, 10);
|
||||
const output = {
|
||||
date: dateStr,
|
||||
scannedAt: now.toISOString(),
|
||||
totalOutliers: allVideos.length,
|
||||
nicheCounts,
|
||||
videos: allVideos,
|
||||
};
|
||||
|
||||
// Save to data/
|
||||
const dataDir = path.join(__dirname, '..', 'data');
|
||||
if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir, { recursive: true });
|
||||
|
||||
const datePath = path.join(dataDir, `${dateStr}.json`);
|
||||
const latestPath = path.join(dataDir, 'latest.json');
|
||||
fs.writeFileSync(datePath, JSON.stringify(output, null, 2));
|
||||
fs.writeFileSync(latestPath, JSON.stringify(output, null, 2));
|
||||
|
||||
console.log(`\n=== Done ===`);
|
||||
console.log(`Total outliers: ${allVideos.length}`);
|
||||
console.log(`Saved to: ${datePath}`);
|
||||
console.log(`Saved to: ${latestPath}`);
|
||||
for (const [k, v] of Object.entries(nicheCounts)) {
|
||||
console.log(` ${v.label}: ${v.count}`);
|
||||
}
|
||||
|
||||
// ── Sync to Prisma DB ───────────────────────────────────────────────────────
|
||||
console.log('\n── Syncing to database…');
|
||||
try {
|
||||
const { Client } = require('pg');
|
||||
// Try .env.production.local first, then .env.local
|
||||
let dbUrl = '';
|
||||
for (const envFile of ['.env.production.local', '.env.local']) {
|
||||
const envPath = path.join(__dirname, '..', envFile);
|
||||
if (!fs.existsSync(envPath)) continue;
|
||||
const envLines = fs.readFileSync(envPath, 'utf-8').split('\n');
|
||||
for (const line of envLines) {
|
||||
const t = line.trim();
|
||||
if (t.startsWith('DATABASE_URL=')) {
|
||||
dbUrl = t.slice('DATABASE_URL='.length).trim().replace(/^["']|["']$/g, '');
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (dbUrl) break;
|
||||
}
|
||||
const client = new Client({ connectionString: dbUrl, ssl: { rejectUnauthorized: false } });
|
||||
await client.connect();
|
||||
const dataJson = JSON.stringify(output);
|
||||
await client.query(
|
||||
`INSERT INTO "DataStore" (key, data, "updatedAt")
|
||||
VALUES ($1, $2::jsonb, NOW())
|
||||
ON CONFLICT (key) DO UPDATE SET data = $2::jsonb, "updatedAt" = NOW()`,
|
||||
['youtube-outliers', dataJson]
|
||||
);
|
||||
await client.end();
|
||||
console.log('✓ Database synced successfully');
|
||||
} catch (err) {
|
||||
console.error('✗ DB sync failed:', err.message);
|
||||
console.error(' (Local files still saved — run sync manually)');
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user