Greg 37f8eac932 Initial commit: CFTC COT Explorer
FastAPI application that ingests CFTC Commitments of Traders data into SQLite
and exposes it via a REST API with analytics endpoints (screener, percentile rank,
concentration). Includes CLI for historical and weekly data ingestion, Docker setup,
and a frontend.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-22 11:23:00 +01:00

626 lines
23 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// ── State ──────────────────────────────────────────────────────────────────
const state = {
view: 'detail',
selectedCode: null,
dateRange: '3Y',
rowType: 'All',
metric: 'noncomm_net',
overlayOI: true,
exchange: '',
compareMarkets: [], // [{cftc_code, name, exchange_abbr, color}]
compareMetric: 'noncomm_net',
compareRange: '3Y',
allCommodities: [], // full list from API
};
// ── API ────────────────────────────────────────────────────────────────────
const API = {
async get(path) {
const r = await fetch(path);
if (!r.ok) throw new Error(`${r.status} ${r.statusText}`);
return r.json();
},
exchanges: () => API.get('/api/exchanges'),
commodities: (exchange) => API.get('/api/commodities' + (exchange ? `?exchange=${exchange}` : '')),
history: (code, fromDate, toDate, rowType) => {
const p = new URLSearchParams({ row_type: rowType });
if (fromDate) p.append('from_date', fromDate);
if (toDate) p.append('to_date', toDate);
return API.get(`/api/positions/${code}/history?${p}`);
},
extremes: (code) => API.get(`/api/positions/${code}/extremes`),
percentile: (code, weeks) => API.get(`/api/analytics/${code}/net-position-percentile?lookback_weeks=${weeks}`),
screener: (exchange, direction, lookback) => {
const p = new URLSearchParams({ lookback_weeks: lookback, top_n: 200 });
if (exchange) p.append('exchange', exchange);
if (direction) p.append('direction', direction === 'long' ? 'long' : 'short');
return API.get(`/api/analytics/screener?${p}`);
},
compare: (codes, metric, fromDate, toDate) => {
const p = new URLSearchParams({ codes: codes.join(','), metric, row_type: 'All' });
if (fromDate) p.append('from_date', fromDate);
if (toDate) p.append('to_date', toDate);
return API.get(`/api/positions/compare?${p}`);
},
};
// ── Utilities ──────────────────────────────────────────────────────────────
function fmt(v) {
if (v === null || v === undefined) return '—';
if (typeof v === 'number') {
if (Math.abs(v) >= 1000000) return (v / 1000000).toFixed(2) + 'M';
if (Math.abs(v) >= 1000) return (v / 1000).toFixed(1) + 'K';
return v.toLocaleString();
}
return v;
}
function fmtPct(v) {
if (v === null || v === undefined) return '—';
return v.toFixed(1) + '%';
}
function fmtChange(v) {
if (v === null || v === undefined) return '—';
const s = (v >= 0 ? '+' : '') + fmt(v);
return s;
}
function dateRangeToFrom(range) {
if (range === 'Max') return null;
const d = new Date();
const years = { '1Y': 1, '3Y': 3, '5Y': 5 }[range] || 3;
d.setFullYear(d.getFullYear() - years);
return d.toISOString().split('T')[0];
}
const CHART_COLORS = [
'#3b82f6', '#22c55e', '#f97316', '#a855f7',
'#ec4899', '#14b8a6', '#eab308', '#ef4444',
];
// ── Charts ─────────────────────────────────────────────────────────────────
let detailChart = null;
let compareChart = null;
function destroyChart(c) { if (c) { c.destroy(); } return null; }
function buildDetailChart(data, metric, overlayOI) {
detailChart = destroyChart(detailChart);
const canvas = document.getElementById('detailChart');
const ctx = canvas.getContext('2d');
const labels = data.map(d => d.report_date);
const metricLabel = document.getElementById('metricSelect').selectedOptions[0]?.text || metric;
let values;
if (metric === 'noncomm_net') {
values = data.map(d => (d.noncomm_long ?? 0) - (d.noncomm_short ?? 0));
} else if (metric === 'comm_net') {
values = data.map(d => (d.comm_long ?? 0) - (d.comm_short ?? 0));
} else {
values = data.map(d => d[metric] ?? null);
}
const datasets = [{
type: 'line',
label: metricLabel,
data: values,
borderColor: '#3b82f6',
backgroundColor: 'rgba(59,130,246,0.08)',
fill: true,
tension: 0.2,
pointRadius: data.length < 60 ? 3 : 0,
pointHoverRadius: 5,
borderWidth: 2,
yAxisID: 'y',
order: 1,
}];
const scales = {
x: {
ticks: { color: '#6b7280', maxTicksLimit: 8, maxRotation: 0 },
grid: { color: '#2d3148' },
},
y: {
position: 'left',
ticks: { color: '#6b7280', callback: v => fmt(v) },
grid: { color: '#2d3148' },
},
};
if (overlayOI) {
datasets.push({
type: 'bar',
label: 'Open Interest',
data: data.map(d => d.open_interest ?? null),
backgroundColor: 'rgba(156,163,175,0.15)',
borderColor: 'rgba(156,163,175,0.3)',
borderWidth: 1,
yAxisID: 'y1',
order: 2,
});
scales.y1 = {
position: 'right',
ticks: { color: '#4b5563', callback: v => fmt(v) },
grid: { drawOnChartArea: false },
};
}
detailChart = new Chart(ctx, {
type: 'bar',
data: { labels, datasets },
options: {
responsive: true,
maintainAspectRatio: false,
interaction: { mode: 'index', intersect: false },
plugins: {
legend: { labels: { color: '#9ca3af', boxWidth: 12 } },
tooltip: {
backgroundColor: '#1a1d27',
borderColor: '#2d3148',
borderWidth: 1,
titleColor: '#e2e8f0',
bodyColor: '#9ca3af',
callbacks: {
label: (ctx) => ` ${ctx.dataset.label}: ${fmt(ctx.parsed.y)}`,
},
},
},
scales,
},
});
}
function buildCompareChart(seriesData, commodities, metric) {
compareChart = destroyChart(compareChart);
const canvas = document.getElementById('compareChart');
const ctx = canvas.getContext('2d');
const allDates = [...new Set(
Object.values(seriesData).flatMap(pts => pts.map(p => p.report_date))
)].sort();
const datasets = state.compareMarkets.map((m, i) => {
const pts = seriesData[m.cftc_code] || [];
const byDate = Object.fromEntries(pts.map(p => [p.report_date, p.value]));
return {
label: m.name,
data: allDates.map(d => byDate[d] ?? null),
borderColor: m.color,
backgroundColor: 'transparent',
tension: 0.2,
pointRadius: allDates.length < 60 ? 3 : 0,
pointHoverRadius: 5,
borderWidth: 2,
spanGaps: false,
};
});
compareChart = new Chart(ctx, {
type: 'line',
data: { labels: allDates, datasets },
options: {
responsive: true,
maintainAspectRatio: false,
interaction: { mode: 'index', intersect: false },
plugins: {
legend: { labels: { color: '#9ca3af', boxWidth: 12 } },
tooltip: {
backgroundColor: '#1a1d27',
borderColor: '#2d3148',
borderWidth: 1,
titleColor: '#e2e8f0',
bodyColor: '#9ca3af',
callbacks: {
label: (ctx) => ` ${ctx.dataset.label}: ${fmt(ctx.parsed.y)}`,
},
},
},
scales: {
x: { ticks: { color: '#6b7280', maxTicksLimit: 8, maxRotation: 0 }, grid: { color: '#2d3148' } },
y: { ticks: { color: '#6b7280', callback: v => fmt(v) }, grid: { color: '#2d3148' } },
},
},
});
}
// ── Market sidebar ─────────────────────────────────────────────────────────
function buildMarketTree(commodities, filter = '') {
const tree = document.getElementById('market-tree');
const groups = {};
const q = filter.toLowerCase();
for (const c of commodities) {
const matches = !q || c.name.toLowerCase().includes(q) || c.exchange_abbr.toLowerCase().includes(q);
if (!matches) continue;
if (!groups[c.exchange_abbr]) groups[c.exchange_abbr] = [];
groups[c.exchange_abbr].push(c);
}
tree.innerHTML = '';
for (const [exch, list] of Object.entries(groups)) {
const grp = document.createElement('div');
grp.className = 'exchange-group';
const lbl = document.createElement('div');
lbl.className = 'exchange-label';
lbl.textContent = exch;
lbl.addEventListener('click', () => {
lbl.classList.toggle('collapsed');
ul.classList.toggle('hidden');
});
const ul = document.createElement('div');
ul.className = 'market-list';
for (const c of list) {
const item = document.createElement('div');
item.className = 'market-item' + (c.cftc_code === state.selectedCode ? ' active' : '');
item.textContent = c.name;
item.title = c.name;
item.dataset.code = c.cftc_code;
item.addEventListener('click', () => selectMarket(c.cftc_code));
ul.appendChild(item);
}
grp.appendChild(lbl);
grp.appendChild(ul);
tree.appendChild(grp);
}
}
// ── Detail view logic ──────────────────────────────────────────────────────
async function selectMarket(code) {
state.selectedCode = code;
document.querySelectorAll('.market-item').forEach(el => {
el.classList.toggle('active', el.dataset.code === code);
});
document.getElementById('detail-placeholder').style.display = 'none';
document.getElementById('detail-content').style.display = 'flex';
document.getElementById('detail-content').style.flexDirection = 'column';
document.getElementById('detail-content').style.gap = '12px';
const meta = state.allCommodities.find(c => c.cftc_code === code);
if (meta) {
document.getElementById('detail-title').textContent = meta.name;
document.getElementById('detail-exchange').textContent = meta.exchange_abbr;
document.getElementById('detail-unit').textContent = meta.contract_unit || '';
}
await refreshDetailChart();
}
async function refreshDetailChart() {
if (!state.selectedCode) return;
const fromDate = dateRangeToFrom(state.dateRange);
try {
const [hist, pctileData] = await Promise.all([
API.history(state.selectedCode, fromDate, null, state.rowType),
API.percentile(state.selectedCode, 156).catch(() => null),
]);
const data = hist.data;
if (!data.length) return;
buildDetailChart(data, state.metric, state.overlayOI);
// Stats bar
const latest = data[data.length - 1];
const metricValues = data.map(d => {
if (state.metric === 'noncomm_net') return (d.noncomm_long ?? 0) - (d.noncomm_short ?? 0);
if (state.metric === 'comm_net') return (d.comm_long ?? 0) - (d.comm_short ?? 0);
return d[state.metric];
}).filter(v => v !== null);
const currentVal = metricValues[metricValues.length - 1];
const maxVal = Math.max(...metricValues);
const minVal = Math.min(...metricValues);
const statCurrent = document.getElementById('statCurrent');
statCurrent.textContent = fmt(currentVal);
statCurrent.className = 'stat-value' + (currentVal > 0 ? ' positive' : currentVal < 0 ? ' negative' : '');
document.getElementById('statMax').textContent = fmt(maxVal);
document.getElementById('statMin').textContent = fmt(minVal);
if (pctileData?.percentile !== null && pctileData?.percentile !== undefined) {
document.getElementById('statPctile').textContent = fmtPct(pctileData.percentile);
} else {
document.getElementById('statPctile').textContent = '—';
}
// Week change for non-comm net
const chgLong = latest.chg_noncomm_long;
const chgShort = latest.chg_noncomm_short;
if (chgLong !== null && chgShort !== null) {
const netChg = chgLong - chgShort;
const el = document.getElementById('statChange');
el.textContent = fmtChange(netChg) + ' net';
el.className = 'stat-value' + (netChg > 0 ? ' positive' : netChg < 0 ? ' negative' : '');
}
} catch (e) {
console.error('Failed to load chart data:', e);
}
}
// ── Screener view ──────────────────────────────────────────────────────────
async function loadScreener() {
const exchange = document.getElementById('screenerExchange').value;
const dirSel = document.getElementById('screenerDirection').value;
const lookback = document.getElementById('screenerLookback').value;
// Map UI direction to API direction
let direction = '';
if (dirSel === 'long') direction = 'long';
else if (dirSel === 'short') direction = 'short';
const tbody = document.getElementById('screenerBody');
tbody.innerHTML = '<tr><td colspan="7" style="text-align:center;padding:20px;color:#6b7280">Loading...</td></tr>';
try {
let rows = await API.screener(exchange, direction, lookback);
// Client-side direction filter (the API `direction` param filters by >=50 or <50,
// but UI wants >=70 and <=30 for "extreme")
if (dirSel === 'long') rows = rows.filter(r => r.pct_rank >= 70);
else if (dirSel === 'short') rows = rows.filter(r => r.pct_rank <= 30);
tbody.innerHTML = '';
for (const row of rows) {
const pct = row.pct_rank;
const pctClass = pct >= 70 ? 'extreme-long' : pct <= 30 ? 'extreme-short' : 'neutral';
const netChg = (row.chg_noncomm_long ?? 0) - (row.chg_noncomm_short ?? 0);
const tr = document.createElement('tr');
tr.innerHTML = `
<td><strong>${row.commodity}</strong></td>
<td><span class="badge">${row.exchange}</span></td>
<td class="num">${fmt(row.noncomm_net)}</td>
<td class="num">${fmt(row.open_interest)}</td>
<td class="num pctile-cell ${pctClass}">
<span class="pctile-bar" style="width:${Math.round((pct || 0) * 0.6)}px"></span>
${pct !== null ? fmtPct(pct) : '—'}
</td>
<td class="num ${row.chg_noncomm_long > 0 ? 'extreme-long' : row.chg_noncomm_long < 0 ? 'extreme-short' : ''}">${fmtChange(row.chg_noncomm_long)}</td>
<td class="num ${row.chg_noncomm_short < 0 ? 'extreme-long' : row.chg_noncomm_short > 0 ? 'extreme-short' : ''}">${fmtChange(row.chg_noncomm_short)}</td>
`;
tr.addEventListener('click', () => {
switchView('detail');
selectMarket(row.cftc_code);
});
tbody.appendChild(tr);
}
if (!rows.length) {
tbody.innerHTML = '<tr><td colspan="7" style="text-align:center;padding:20px;color:#6b7280">No markets match the current filters.</td></tr>';
}
} catch (e) {
tbody.innerHTML = `<tr><td colspan="7" style="text-align:center;padding:20px;color:#ef4444">Error: ${e.message}</td></tr>`;
}
}
// ── Compare view ───────────────────────────────────────────────────────────
function renderCompareTags() {
const container = document.getElementById('compareTags');
container.innerHTML = '';
state.compareMarkets.forEach((m, i) => {
const tag = document.createElement('div');
tag.className = 'compare-tag';
tag.style.background = m.color;
tag.innerHTML = `${m.name} <button class="compare-tag-remove" data-code="${m.cftc_code}">×</button>`;
tag.querySelector('button').addEventListener('click', (e) => {
e.stopPropagation();
removeCompareMarket(m.cftc_code);
});
container.appendChild(tag);
});
}
function addCompareMarket(commodity) {
if (state.compareMarkets.find(m => m.cftc_code === commodity.cftc_code)) return;
if (state.compareMarkets.length >= 8) return;
state.compareMarkets.push({
...commodity,
color: CHART_COLORS[state.compareMarkets.length % CHART_COLORS.length],
});
renderCompareTags();
loadCompareChart();
}
function removeCompareMarket(code) {
state.compareMarkets = state.compareMarkets.filter(m => m.cftc_code !== code);
// Reassign colors
state.compareMarkets.forEach((m, i) => { m.color = CHART_COLORS[i % CHART_COLORS.length]; });
renderCompareTags();
loadCompareChart();
}
async function loadCompareChart() {
const placeholder = document.getElementById('comparePlaceholder');
const canvas = document.getElementById('compareChart');
if (!state.compareMarkets.length) {
placeholder.style.display = 'flex';
canvas.style.display = 'none';
compareChart = destroyChart(compareChart);
return;
}
placeholder.style.display = 'none';
canvas.style.display = 'block';
const fromDate = dateRangeToFrom(state.compareRange);
const codes = state.compareMarkets.map(m => m.cftc_code);
try {
const resp = await API.compare(codes, state.compareMetric, fromDate, null);
buildCompareChart(resp.series, resp.commodities, state.compareMetric);
} catch (e) {
console.error('Compare chart error:', e);
}
}
// ── Compare autocomplete ───────────────────────────────────────────────────
function setupCompareSearch() {
const input = document.getElementById('compareSearch');
const dropdown = document.getElementById('compareDropdown');
input.addEventListener('input', () => {
const q = input.value.trim().toLowerCase();
if (!q) { dropdown.style.display = 'none'; return; }
const matches = state.allCommodities
.filter(c => c.name.toLowerCase().includes(q) || c.exchange_abbr.toLowerCase().includes(q))
.slice(0, 10);
dropdown.innerHTML = '';
if (!matches.length) { dropdown.style.display = 'none'; return; }
matches.forEach(c => {
const item = document.createElement('div');
item.className = 'autocomplete-item';
item.innerHTML = `${c.name} <span class="autocomplete-item-exchange">${c.exchange_abbr}</span>`;
item.addEventListener('click', () => {
addCompareMarket(c);
input.value = '';
dropdown.style.display = 'none';
});
dropdown.appendChild(item);
});
dropdown.style.display = 'block';
});
document.addEventListener('click', (e) => {
if (!input.contains(e.target) && !dropdown.contains(e.target)) {
dropdown.style.display = 'none';
}
});
}
// ── View switching ─────────────────────────────────────────────────────────
function switchView(view) {
state.view = view;
document.querySelectorAll('.view').forEach(el => { el.style.display = 'none'; });
document.querySelectorAll('.tab-btn').forEach(btn => {
btn.classList.toggle('active', btn.dataset.view === view);
});
document.getElementById(`view-${view}`).style.display = view === 'detail' ? 'flex' : 'flex';
document.getElementById(`view-${view}`).style.flexDirection = view === 'detail' ? 'row' : 'column';
if (view === 'screener') loadScreener();
if (view === 'compare') renderCompareTags();
}
// ── Populate exchange dropdowns ────────────────────────────────────────────
async function populateExchangeDropdowns(exchanges) {
['exchangeFilter', 'screenerExchange'].forEach(id => {
const sel = document.getElementById(id);
exchanges.forEach(e => {
const opt = document.createElement('option');
opt.value = e.exchange_abbr;
opt.textContent = `${e.exchange_abbr} (${e.commodity_count})`;
sel.appendChild(opt);
});
});
}
// ── Init ───────────────────────────────────────────────────────────────────
async function init() {
try {
const [exchanges, commodities] = await Promise.all([
API.exchanges(),
API.commodities(),
]);
state.allCommodities = commodities;
await populateExchangeDropdowns(exchanges);
buildMarketTree(commodities);
// Select first commodity by default
if (commodities.length > 0) {
selectMarket(commodities[0].cftc_code);
}
} catch (e) {
console.error('Init failed:', e);
}
}
// ── Event listeners ────────────────────────────────────────────────────────
document.addEventListener('DOMContentLoaded', () => {
// Tab switching
document.querySelectorAll('.tab-btn').forEach(btn => {
btn.addEventListener('click', () => switchView(btn.dataset.view));
});
// Exchange filter (sidebar)
document.getElementById('exchangeFilter').addEventListener('change', async (e) => {
state.exchange = e.target.value;
const filtered = state.allCommodities.filter(c =>
!state.exchange || c.exchange_abbr === state.exchange
);
buildMarketTree(filtered);
});
// Market search
document.getElementById('marketSearch').addEventListener('input', (e) => {
const q = e.target.value;
const filtered = state.allCommodities.filter(c =>
!state.exchange || c.exchange_abbr === state.exchange
);
buildMarketTree(filtered, q);
});
// Date range buttons
document.querySelectorAll('.range-btn').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('.range-btn').forEach(b => b.classList.remove('active'));
btn.classList.add('active');
state.dateRange = btn.dataset.range;
refreshDetailChart();
});
});
// Row type buttons
document.querySelectorAll('.rt-btn').forEach(btn => {
btn.addEventListener('click', () => {
document.querySelectorAll('.rt-btn').forEach(b => b.classList.remove('active'));
btn.classList.add('active');
state.rowType = btn.dataset.rt;
refreshDetailChart();
});
});
// Metric select
document.getElementById('metricSelect').addEventListener('change', (e) => {
state.metric = e.target.value;
refreshDetailChart();
});
// OI overlay checkbox
document.getElementById('overlayOI').addEventListener('change', (e) => {
state.overlayOI = e.target.checked;
refreshDetailChart();
});
// Screener
document.getElementById('screenerRefresh').addEventListener('click', loadScreener);
document.getElementById('screenerDirection').addEventListener('change', loadScreener);
document.getElementById('screenerLookback').addEventListener('change', loadScreener);
document.getElementById('screenerExchange').addEventListener('change', loadScreener);
// Compare
document.getElementById('compareMetric').addEventListener('change', (e) => {
state.compareMetric = e.target.value;
loadCompareChart();
});
document.getElementById('compareRange').addEventListener('change', (e) => {
state.compareRange = e.target.value;
loadCompareChart();
});
setupCompareSearch();
init();
});