373 lines
12 KiB
HTML
373 lines
12 KiB
HTML
<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
|
<title>CSV File Search</title>
|
|
<style>
|
|
body { font-family: system-ui, Arial, sans-serif; max-width: 900px; margin: 24px auto; padding: 0 12px; }
|
|
.row { display: flex; gap: 10px; align-items: center; flex-wrap: wrap; }
|
|
input[type="text"] { flex: 1; min-width: 260px; padding: 10px; }
|
|
button { padding: 10px 14px; cursor: pointer; }
|
|
pre { background: #f6f7f9; padding: 12px; overflow: auto; border-radius: 8px; }
|
|
.meta { color: #555; margin-top: 6px; font-size: 0.95em; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>CSV File Search</h1>
|
|
|
|
<div class="row">
|
|
<label>
|
|
<div style="margin-bottom:6px;">Upload CSV files</div>
|
|
<input id="csvFiles" type="file" accept=".csv,text/csv" multiple />
|
|
</label>
|
|
|
|
<button id="loadBtn" disabled>Load CSVs</button>
|
|
</div>
|
|
|
|
<div class="meta" id="status">Status: waiting for files.</div>
|
|
|
|
<hr style="margin: 18px 0;" />
|
|
|
|
<div class="row">
|
|
<input id="queryBox" type="text" placeholder="Type a search term (e.g., john)..." disabled />
|
|
<button id="searchBtn" disabled>Search</button>
|
|
<button id="listColumnsBtn" disabled>List columns</button>
|
|
</div>
|
|
|
|
<p class="meta" id="hint">
|
|
Searches all “text-like” columns in all loaded CSVs using a substring match.
|
|
</p>
|
|
|
|
<h2 style="margin-top: 18px;">Results</h2>
|
|
<pre id="out">No results yet.</pre>
|
|
|
|
<script src="https://cdn.jsdelivr.net/npm/papaparse@5.4.1/papaparse.min.js"></script>
|
|
<script>
|
|
// datasets: array of { file, fileName, headers, textTargets:Set(headerName) }
|
|
let datasets = [];
|
|
|
|
const statusEl = document.getElementById('status');
|
|
const outEl = document.getElementById('out');
|
|
const csvFilesEl = document.getElementById('csvFiles');
|
|
const loadBtn = document.getElementById('loadBtn');
|
|
const queryBox = document.getElementById('queryBox');
|
|
const searchBtn = document.getElementById('searchBtn');
|
|
const listColumnsBtn = document.getElementById('listColumnsBtn');
|
|
|
|
function setStatus(msg) { statusEl.textContent = "Status: " + msg; }
|
|
|
|
function detectTextLikeColumnsFromSample(sampleRows, headers, sampleSize = 200) {
|
|
const limit = Math.min(sampleRows.length, sampleSize);
|
|
|
|
// Stats indexed by column index (so duplicates/empties don't break alignment)
|
|
const colStats = new Array(headers.length).fill(null).map(() => ({ nonEmpty: 0, numericLike: 0 }));
|
|
|
|
const numericRegex = /^-?\d+(\.\d+)?$/;
|
|
|
|
for (let r = 0; r < limit; r++) {
|
|
const row = sampleRows[r];
|
|
|
|
for (let c = 0; c < headers.length; c++) {
|
|
const h = headers[c];
|
|
if (!h) continue; // ignore empty header names for "textTargets" selection
|
|
|
|
const v = (row[c] ?? '').toString().trim();
|
|
if (!v) continue;
|
|
|
|
colStats[c].nonEmpty++;
|
|
if (numericRegex.test(v)) colStats[c].numericLike++;
|
|
}
|
|
}
|
|
|
|
// Keep a Set of header names for UI and matching selection (matches may include duplicate names)
|
|
const targets = new Set();
|
|
|
|
for (let c = 0; c < headers.length; c++) {
|
|
const h = headers[c];
|
|
if (!h) continue;
|
|
|
|
const s = colStats[c];
|
|
if (s.nonEmpty === 0) continue;
|
|
|
|
const numericRatio = s.numericLike / Math.max(1, s.nonEmpty);
|
|
// numericRatio < 0.8 => not mostly-numeric => treat as text-like
|
|
if (numericRatio < 0.8) targets.add(h);
|
|
}
|
|
|
|
return targets;
|
|
}
|
|
|
|
csvFilesEl.addEventListener('change', () => {
|
|
loadBtn.disabled = !csvFilesEl.files?.length;
|
|
});
|
|
|
|
loadBtn.addEventListener('click', async () => {
|
|
const files = [...(csvFilesEl.files || [])];
|
|
if (!files.length) return;
|
|
|
|
try {
|
|
outEl.textContent = "Loading CSVs...";
|
|
setStatus(`loading ${files.length} file(s)...`);
|
|
datasets = [];
|
|
|
|
// Load sequentially so you don't spawn multiple heavy parses at once
|
|
const SAMPLE_SIZE = 200;
|
|
|
|
for (let fi = 0; fi < files.length; fi++) {
|
|
const file = files[fi];
|
|
setStatus(`sampling ${fi + 1}/${files.length}: ${file.name}`);
|
|
outEl.textContent = `Sampling: ${file.name}`;
|
|
|
|
// Pass 1: stream to get headers + first N data rows
|
|
const sampleRows = [];
|
|
let headers = null;
|
|
let headerSet = false;
|
|
|
|
await new Promise((resolve, reject) => {
|
|
Papa.parse(file, {
|
|
header: false,
|
|
skipEmptyLines: true,
|
|
worker: true,
|
|
chunkSize: 1024 * 1024,
|
|
step: function(results, parser) {
|
|
if (!headerSet) {
|
|
// First row is header row
|
|
headers = results.data.map(h => (h ?? '').toString().trim());
|
|
// IMPORTANT: do NOT filter empty header names; keep positions stable
|
|
headerSet = true;
|
|
return;
|
|
}
|
|
|
|
if (headers && sampleRows.length < SAMPLE_SIZE) {
|
|
sampleRows.push(results.data);
|
|
}
|
|
|
|
if (sampleRows.length >= SAMPLE_SIZE) {
|
|
parser.abort(); // only need a sample for column detection
|
|
}
|
|
},
|
|
complete: function() {
|
|
if (!headers || !headers.length) return resolve();
|
|
const textTargets = detectTextLikeColumnsFromSample(sampleRows, headers, SAMPLE_SIZE);
|
|
datasets.push({ file, fileName: file.name, headers, textTargets });
|
|
resolve();
|
|
},
|
|
error: function(err) {
|
|
reject(err);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
queryBox.disabled = false;
|
|
searchBtn.disabled = false;
|
|
listColumnsBtn.disabled = false;
|
|
|
|
const totalCols = datasets.reduce((acc, d) => acc + d.textTargets.size, 0);
|
|
setStatus("loaded.");
|
|
outEl.textContent =
|
|
`Loaded ${datasets.length} CSV(s). Total text-like columns (sum of unique names): ${totalCols}.`;
|
|
} catch (e) {
|
|
console.error(e);
|
|
setStatus("failed.");
|
|
outEl.textContent = "Error:\n" + e.message;
|
|
}
|
|
});
|
|
|
|
listColumnsBtn.addEventListener('click', () => {
|
|
if (!datasets.length) return;
|
|
|
|
const lines = [];
|
|
for (const d of datasets) {
|
|
const cols = [...d.textTargets];
|
|
if (!cols.length) continue;
|
|
lines.push(`${d.fileName} (${cols.length}):\n- ${cols.join("\n- ")}`);
|
|
}
|
|
|
|
if (!lines.length) {
|
|
outEl.textContent = "No text-like columns detected in any loaded CSVs.";
|
|
return;
|
|
}
|
|
|
|
outEl.textContent = "Text-like columns:\n\n" + lines.join("\n\n");
|
|
});
|
|
|
|
searchBtn.addEventListener('click', async () => {
|
|
const term = (queryBox.value || '').trim();
|
|
if (!term) {
|
|
outEl.textContent = "Type a search term first.";
|
|
return;
|
|
}
|
|
|
|
if (!datasets.length) return;
|
|
|
|
const termLower = term.toLowerCase();
|
|
const LIMIT = 1000;
|
|
const results = [];
|
|
|
|
// Total search time only
|
|
const t0 = performance.now();
|
|
|
|
// Build a quick lookup for headers by file name
|
|
const headersByFile = new Map(datasets.map(d => [d.fileName, d.headers]));
|
|
|
|
// Helper: count data rows (excluding header) for progress.
|
|
// Uses parsing steps to count rows precisely.
|
|
async function countDataRows(file) {
|
|
return await new Promise((resolve, reject) => {
|
|
let sawHeader = false;
|
|
let dataRows = 0;
|
|
|
|
Papa.parse(file, {
|
|
header: false,
|
|
skipEmptyLines: true,
|
|
worker: true,
|
|
chunkSize: 1024 * 1024,
|
|
step: function() {
|
|
if (!sawHeader) {
|
|
sawHeader = true;
|
|
} else {
|
|
dataRows++;
|
|
}
|
|
},
|
|
complete: function() {
|
|
resolve(dataRows);
|
|
},
|
|
error: function(err) {
|
|
reject(err);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
try {
|
|
setStatus(`searching for "${term}"...`);
|
|
outEl.textContent = `Searching: "${term}"\n\nProgress: 0%`;
|
|
|
|
// Precompute total rows across all loaded CSVs (sum of data rows) for global percentage
|
|
const totalRowsPerFile = [];
|
|
let grandTotalRows = 0;
|
|
|
|
for (let di = 0; di < datasets.length; di++) {
|
|
const d = datasets[di];
|
|
if (!d.textTargets.size) {
|
|
totalRowsPerFile.push(0);
|
|
continue;
|
|
}
|
|
setStatus(`counting rows ${di + 1}/${datasets.length}: ${d.fileName}`);
|
|
outEl.textContent = `Searching: "${term}"\n\nProgress: 0% (counting rows in ${di + 1}/${datasets.length})`;
|
|
const cnt = await countDataRows(d.file);
|
|
totalRowsPerFile.push(cnt);
|
|
grandTotalRows += cnt;
|
|
}
|
|
|
|
// Avoid divide-by-zero
|
|
grandTotalRows = Math.max(1, grandTotalRows);
|
|
|
|
let processedRowsGlobal = 0;
|
|
|
|
for (let di = 0; di < datasets.length; di++) {
|
|
const d = datasets[di];
|
|
if (!d.textTargets.size) continue;
|
|
|
|
const fileTotalRows = totalRowsPerFile[di] || 0;
|
|
|
|
setStatus(`searching ${di + 1}/${datasets.length}: ${d.fileName}`);
|
|
outEl.textContent =
|
|
`Searching: "${term}"\n\nProgress: ${Math.floor((processedRowsGlobal / grandTotalRows) * 100)}% (processed ${processedRowsGlobal}/${grandTotalRows})`;
|
|
|
|
// Build candidate indices by header position (supports duplicate header names)
|
|
const targetIndices = [];
|
|
for (let i = 0; i < d.headers.length; i++) {
|
|
const h = d.headers[i];
|
|
if (h && d.textTargets.has(h)) targetIndices.push(i);
|
|
}
|
|
|
|
let dataRowIndex = 0;
|
|
let sawHeader = false;
|
|
|
|
await new Promise((resolve, reject) => {
|
|
Papa.parse(d.file, {
|
|
header: false,
|
|
skipEmptyLines: true,
|
|
worker: true,
|
|
chunkSize: 1024 * 1024,
|
|
step: function(results2, parser) {
|
|
if (!sawHeader) {
|
|
sawHeader = true;
|
|
return; // skip header row
|
|
}
|
|
|
|
// This is a data row (since we skipped the header row above)
|
|
dataRowIndex++;
|
|
|
|
// Update progress (global)
|
|
processedRowsGlobal++;
|
|
const pct = Math.floor((processedRowsGlobal / grandTotalRows) * 100);
|
|
|
|
outEl.textContent =
|
|
`Searching: "${term}"\n\nProgress: ${pct}% (processed ${processedRowsGlobal}/${grandTotalRows})`;
|
|
|
|
// If any target cell in this row matches, store the whole row once
|
|
let rowMatched = false;
|
|
|
|
for (const ci of targetIndices) {
|
|
const v = (results2.data?.[ci] ?? '').toString().trim();
|
|
if (v && v.toLowerCase().includes(termLower)) {
|
|
rowMatched = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (rowMatched) {
|
|
results.push({
|
|
file_name: d.fileName,
|
|
row_index: dataRowIndex + 1, // header row is row 1; data starts at 2
|
|
row: results2.data
|
|
});
|
|
|
|
if (results.length >= LIMIT) {
|
|
parser.abort();
|
|
}
|
|
}
|
|
},
|
|
complete: function() { resolve(); },
|
|
error: function(err) { reject(err); }
|
|
});
|
|
});
|
|
|
|
if (results.length >= LIMIT) break;
|
|
}
|
|
|
|
if (!results.length) {
|
|
outEl.textContent = `No matches found for "${term}".`;
|
|
} else {
|
|
const lines = results.map(x => {
|
|
const headers = headersByFile.get(x.file_name) || [];
|
|
const pairs = x.row.map((val, i) => {
|
|
const key = headers[i] ?? `col_${i + 1}`;
|
|
const safeVal = (val ?? '').toString();
|
|
return `${key}: ${safeVal}`;
|
|
});
|
|
return `${x.file_name} (row ${x.row_index}):\n ${pairs.join("\n ")}`;
|
|
});
|
|
|
|
outEl.textContent =
|
|
`Found ${results.length} matched row(s) for "${term}" (showing up to ${LIMIT}):\n\n` +
|
|
lines.join("\n\n");
|
|
}
|
|
|
|
const t1 = performance.now();
|
|
setStatus(`done. Search time: ${(t1 - t0).toFixed(0)} ms`);
|
|
} catch (e) {
|
|
console.error(e);
|
|
setStatus("failed.");
|
|
outEl.textContent = "Error:\n" + e.message;
|
|
}
|
|
});
|
|
</script>
|
|
</body>
|
|
</html>
|
|
|