296 lines
10 KiB
HTML
296 lines
10 KiB
HTML
|
<!doctype html>
|
||
|
|
<html lang="en">
|
||
|
|
<head>
|
||
|
|
<meta charset="utf-8" />
|
||
|
|
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||
|
|
<title>CSV File Search</title>
|
||
|
|
<style>
|
||
|
|
body { font-family: system-ui, Arial, sans-serif; max-width: 900px; margin: 24px auto; padding: 0 12px; }
|
||
|
|
.row { display: flex; gap: 10px; align-items: center; flex-wrap: wrap; }
|
||
|
|
input[type="text"] { flex: 1; min-width: 260px; padding: 10px; }
|
||
|
|
button { padding: 10px 14px; cursor: pointer; }
|
||
|
|
pre { background: #f6f7f9; padding: 12px; overflow: auto; border-radius: 8px; }
|
||
|
|
.meta { color: #555; margin-top: 6px; font-size: 0.95em; }
|
||
|
|
</style>
|
||
|
|
</head>
|
||
|
|
<body>
|
||
|
|
<h1>CSV File Search</h1>
|
||
|
|
|
||
|
|
<div class="row">
|
||
|
|
<label>
|
||
|
|
<div style="margin-bottom:6px;">Upload CSV files</div>
|
||
|
|
<input id="csvFiles" type="file" accept=".csv,text/csv" multiple />
|
||
|
|
</label>
|
||
|
|
|
||
|
|
<button id="loadBtn" disabled>Load CSVs</button>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
<div class="meta" id="status">Status: waiting for files.</div>
|
||
|
|
|
||
|
|
<hr style="margin: 18px 0;" />
|
||
|
|
|
||
|
|
<div class="row">
|
||
|
|
<input id="queryBox" type="text" placeholder="Type a search term (e.g., john)..." disabled />
|
||
|
|
<button id="searchBtn" disabled>Search</button>
|
||
|
|
<button id="listColumnsBtn" disabled>List columns</button>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
<p class="meta" id="hint">
|
||
|
|
Searches all “text-like” columns in all loaded CSVs using a substring match.
|
||
|
|
</p>
|
||
|
|
|
||
|
|
<h2 style="margin-top: 18px;">Results</h2>
|
||
|
|
<pre id="out">No results yet.</pre>
|
||
|
|
|
||
|
|
<script src="https://cdn.jsdelivr.net/npm/papaparse@5.4.1/papaparse.min.js"></script>
|
||
|
|
<script>
|
||
|
|
// datasets: array of { file, fileName, headers, textTargets:Set(headerName) }
|
||
|
|
let datasets = [];
|
||
|
|
|
||
|
|
const statusEl = document.getElementById('status');
|
||
|
|
const outEl = document.getElementById('out');
|
||
|
|
const csvFilesEl = document.getElementById('csvFiles');
|
||
|
|
const loadBtn = document.getElementById('loadBtn');
|
||
|
|
const queryBox = document.getElementById('queryBox');
|
||
|
|
const searchBtn = document.getElementById('searchBtn');
|
||
|
|
const listColumnsBtn = document.getElementById('listColumnsBtn');
|
||
|
|
|
||
|
|
function setStatus(msg) { statusEl.textContent = "Status: " + msg; }
|
||
|
|
|
||
|
|
function detectTextLikeColumnsFromSample(sampleRows, headers, sampleSize = 200) {
|
||
|
|
const limit = Math.min(sampleRows.length, sampleSize);
|
||
|
|
|
||
|
|
// Stats indexed by column index (so duplicates/empties don't break alignment)
|
||
|
|
const colStats = new Array(headers.length).fill(null).map(() => ({ nonEmpty: 0, numericLike: 0 }));
|
||
|
|
|
||
|
|
const numericRegex = /^-?\d+(\.\d+)?$/;
|
||
|
|
|
||
|
|
for (let r = 0; r < limit; r++) {
|
||
|
|
const row = sampleRows[r];
|
||
|
|
|
||
|
|
for (let c = 0; c < headers.length; c++) {
|
||
|
|
const h = headers[c];
|
||
|
|
if (!h) continue; // ignore empty header names for "textTargets" selection
|
||
|
|
|
||
|
|
const v = (row[c] ?? '').toString().trim();
|
||
|
|
if (!v) continue;
|
||
|
|
|
||
|
|
colStats[c].nonEmpty++;
|
||
|
|
if (numericRegex.test(v)) colStats[c].numericLike++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Keep a Set of header names for UI and matching selection (matches may include duplicate names)
|
||
|
|
const targets = new Set();
|
||
|
|
|
||
|
|
for (let c = 0; c < headers.length; c++) {
|
||
|
|
const h = headers[c];
|
||
|
|
if (!h) continue;
|
||
|
|
|
||
|
|
const s = colStats[c];
|
||
|
|
if (s.nonEmpty === 0) continue;
|
||
|
|
|
||
|
|
const numericRatio = s.numericLike / Math.max(1, s.nonEmpty);
|
||
|
|
// numericRatio < 0.8 => not mostly-numeric => treat as text-like
|
||
|
|
if (numericRatio < 0.8) targets.add(h);
|
||
|
|
}
|
||
|
|
|
||
|
|
return targets;
|
||
|
|
}
|
||
|
|
|
||
|
|
csvFilesEl.addEventListener('change', () => {
|
||
|
|
loadBtn.disabled = !csvFilesEl.files?.length;
|
||
|
|
});
|
||
|
|
|
||
|
|
loadBtn.addEventListener('click', async () => {
|
||
|
|
const files = [...(csvFilesEl.files || [])];
|
||
|
|
if (!files.length) return;
|
||
|
|
|
||
|
|
try {
|
||
|
|
outEl.textContent = "Loading CSVs...";
|
||
|
|
setStatus(`loading ${files.length} file(s)...`);
|
||
|
|
datasets = [];
|
||
|
|
|
||
|
|
// Load sequentially so you don't spawn multiple heavy parses at once
|
||
|
|
const SAMPLE_SIZE = 200;
|
||
|
|
|
||
|
|
for (let fi = 0; fi < files.length; fi++) {
|
||
|
|
const file = files[fi];
|
||
|
|
setStatus(`sampling ${fi + 1}/${files.length}: ${file.name}`);
|
||
|
|
outEl.textContent = `Sampling: ${file.name}`;
|
||
|
|
|
||
|
|
// Pass 1: stream to get headers + first N data rows
|
||
|
|
const sampleRows = [];
|
||
|
|
let headers = null;
|
||
|
|
let headerSet = false;
|
||
|
|
|
||
|
|
await new Promise((resolve, reject) => {
|
||
|
|
Papa.parse(file, {
|
||
|
|
header: false,
|
||
|
|
skipEmptyLines: true,
|
||
|
|
worker: true,
|
||
|
|
chunkSize: 1024 * 1024,
|
||
|
|
step: function(results, parser) {
|
||
|
|
if (!headerSet) {
|
||
|
|
// First row is header row
|
||
|
|
headers = results.data.map(h => (h ?? '').toString().trim());
|
||
|
|
// IMPORTANT: do NOT filter empty header names; keep positions stable
|
||
|
|
headerSet = true;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (headers && sampleRows.length < SAMPLE_SIZE) {
|
||
|
|
sampleRows.push(results.data);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (sampleRows.length >= SAMPLE_SIZE) {
|
||
|
|
parser.abort(); // only need a sample for column detection
|
||
|
|
}
|
||
|
|
},
|
||
|
|
complete: function() {
|
||
|
|
if (!headers || !headers.length) return resolve();
|
||
|
|
const textTargets = detectTextLikeColumnsFromSample(sampleRows, headers, SAMPLE_SIZE);
|
||
|
|
datasets.push({ file, fileName: file.name, headers, textTargets });
|
||
|
|
resolve();
|
||
|
|
},
|
||
|
|
error: function(err) {
|
||
|
|
reject(err);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
queryBox.disabled = false;
|
||
|
|
searchBtn.disabled = false;
|
||
|
|
listColumnsBtn.disabled = false;
|
||
|
|
|
||
|
|
const totalCols = datasets.reduce((acc, d) => acc + d.textTargets.size, 0);
|
||
|
|
setStatus("loaded.");
|
||
|
|
outEl.textContent =
|
||
|
|
`Loaded ${datasets.length} CSV(s). Total text-like columns (sum of unique names): ${totalCols}.`;
|
||
|
|
} catch (e) {
|
||
|
|
console.error(e);
|
||
|
|
setStatus("failed.");
|
||
|
|
outEl.textContent = "Error:\n" + e.message;
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
listColumnsBtn.addEventListener('click', () => {
|
||
|
|
if (!datasets.length) return;
|
||
|
|
|
||
|
|
const lines = [];
|
||
|
|
for (const d of datasets) {
|
||
|
|
const cols = [...d.textTargets];
|
||
|
|
if (!cols.length) continue;
|
||
|
|
lines.push(`${d.fileName} (${cols.length}):\n- ${cols.join("\n- ")}`);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!lines.length) {
|
||
|
|
outEl.textContent = "No text-like columns detected in any loaded CSV.";
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
outEl.textContent = "Text-like columns:\n\n" + lines.join("\n\n");
|
||
|
|
});
|
||
|
|
|
||
|
|
searchBtn.addEventListener('click', async () => {
|
||
|
|
const term = (queryBox.value || '').trim();
|
||
|
|
if (!term) {
|
||
|
|
outEl.textContent = "Type a search term first.";
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!datasets.length) return;
|
||
|
|
|
||
|
|
const termLower = term.toLowerCase();
|
||
|
|
const LIMIT = 1000;
|
||
|
|
const results = [];
|
||
|
|
|
||
|
|
try {
|
||
|
|
setStatus(`searching for "${term}"...`);
|
||
|
|
outEl.textContent = `Searching: "${term}"`;
|
||
|
|
|
||
|
|
// Search each dataset by streaming the entire file (second pass per file)
|
||
|
|
for (let di = 0; di < datasets.length; di++) {
|
||
|
|
const d = datasets[di];
|
||
|
|
if (!d.textTargets.size) continue;
|
||
|
|
|
||
|
|
setStatus(`searching ${di + 1}/${datasets.length}: ${d.fileName}`);
|
||
|
|
|
||
|
|
// Build candidate indices by header position (supports duplicate header names)
|
||
|
|
const targetIndices = [];
|
||
|
|
for (let i = 0; i < d.headers.length; i++) {
|
||
|
|
const h = d.headers[i];
|
||
|
|
if (h && d.textTargets.has(h)) targetIndices.push(i);
|
||
|
|
}
|
||
|
|
|
||
|
|
let dataRowIndex = 0;
|
||
|
|
let sawHeader = false;
|
||
|
|
|
||
|
|
await new Promise((resolve, reject) => {
|
||
|
|
Papa.parse(d.file, {
|
||
|
|
header: false,
|
||
|
|
skipEmptyLines: true,
|
||
|
|
worker: true,
|
||
|
|
chunkSize: 1024 * 1024,
|
||
|
|
step: function(results2, parser) {
|
||
|
|
if (!sawHeader) {
|
||
|
|
sawHeader = true;
|
||
|
|
return; // skip header row
|
||
|
|
}
|
||
|
|
|
||
|
|
// This is a data row (since we skipped the header row above)
|
||
|
|
dataRowIndex++;
|
||
|
|
|
||
|
|
// IMPORTANT: collect ALL matches across candidate columns for this row
|
||
|
|
for (const ci of targetIndices) {
|
||
|
|
const v = (results2.data?.[ci] ?? '').toString().trim();
|
||
|
|
if (v && v.toLowerCase().includes(termLower)) {
|
||
|
|
results.push({
|
||
|
|
file_name: d.fileName,
|
||
|
|
row_index: dataRowIndex + 1, // header row is row 1; data starts at 2
|
||
|
|
column_name: d.headers[ci], // may be "" if header was empty
|
||
|
|
match_value: v
|
||
|
|
});
|
||
|
|
|
||
|
|
if (results.length >= LIMIT) {
|
||
|
|
parser.abort();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
complete: function() { resolve(); },
|
||
|
|
error: function(err) { reject(err); }
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
if (results.length >= LIMIT) break;
|
||
|
|
|
||
|
|
// Update UI occasionally between files
|
||
|
|
outEl.textContent =
|
||
|
|
`Searching: "${term}"\n\nCollected ${results.length} match(es) so far (up to ${LIMIT}).`;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!results.length) {
|
||
|
|
outEl.textContent = `No matches found for "${term}".`;
|
||
|
|
} else {
|
||
|
|
const lines = results.map(x =>
|
||
|
|
`${x.file_name} (row ${x.row_index}): ${x.column_name} = ${x.match_value}`
|
||
|
|
);
|
||
|
|
outEl.textContent =
|
||
|
|
`Found ${results.length} match(es) for "${term}" (showing up to ${LIMIT}):\n\n` + lines.join("\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
setStatus("done.");
|
||
|
|
} catch (e) {
|
||
|
|
console.error(e);
|
||
|
|
setStatus("failed.");
|
||
|
|
outEl.textContent = "Error:\n" + e.message;
|
||
|
|
}
|
||
|
|
});
|
||
|
|
</script>
|
||
|
|
</body>
|
||
|
|
</html>
|
||
|
|
|