loadWindow($filePath, $startRow, $endRow); foreach ($window['rows'] as $rowIndex => $row) { $normalizedRow = []; foreach ($row as $columnLetter => $cellValue) { $normalizedRow[$columnLetter] = $this->normalizeHeader($cellValue); } if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) { continue; } $headers = []; $headerLookup = []; foreach ($normalizedRow as $columnLetter => $headerValue) { if ($headerValue === '' || $headerValue === 'Revenue Type') { continue; } $headers[] = $headerValue; $headerLookup[$columnLetter] = $headerValue; } return [ 'header_row_index' => (int) $rowIndex, 'headers' => $headers, 'header_lookup' => $headerLookup, 'highest_data_row' => (int) $window['highest_data_row'], ]; } if ((int) $window['highest_data_row'] < $endRow) { break; } } throw new RuntimeException('Could not locate the header row in the uploaded workbook.'); } public function readChunk(string $filePath, int $startRow, int $chunkSize, array $headerLookup): array { $endRow = $startRow + $chunkSize - 1; $window = $this->loadWindow($filePath, $startRow, $endRow); $rows = []; foreach ($window['rows'] as $rowIndex => $row) { $mappedRow = []; foreach ($headerLookup as $columnLetter => $header) { $mappedRow[$header] = $this->normalizeCellValue($row[$columnLetter] ?? null); } if ($this->isBlankRow($mappedRow)) { continue; } $rows[] = [ 'row_number' => (int) $rowIndex, 'data' => $mappedRow, ]; } return [ 'rows' => $rows, 'highest_data_row' => (int) $window['highest_data_row'], ]; } public function read(string $filePath, array $expectedHeaders = []): array { $header = $this->detectHeader($filePath, $expectedHeaders); $allRows = []; $nextRow = $header['header_row_index'] + 1; while ($nextRow <= $header['highest_data_row']) { $chunk = $this->readChunk($filePath, $nextRow, 500, $header['header_lookup']); $allRows = array_merge($allRows, $chunk['rows']); $nextRow += 500; } return [ 'headers' => $header['headers'], 'rows' => $allRows, ]; } public function buildChunkCache(string $filePath, string $cacheBasePath, array $expectedHeaders = [], int $chunkSize = 50): array { if ($chunkSize < 1) { throw new RuntimeException('Chunk size must be at least 1.'); } $reader = $this->createSpoutReader($filePath); $metaPath = $this->cacheMetaPath($cacheBasePath); $rowsPath = $this->cacheRowsPath($cacheBasePath); $cacheDir = dirname($metaPath); if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) { throw new RuntimeException('Unable to create the spreadsheet cache directory.'); } $rowsHandle = fopen($rowsPath, 'wb'); if ($rowsHandle === false) { throw new RuntimeException('Unable to create the spreadsheet cache file.'); } $headerFound = false; $rowIndex = 0; $chunkRows = []; $chunkOffsets = []; $totalRows = 0; $headerRowIndex = 0; $headers = []; $headerLookup = []; try { $reader->open($filePath); foreach ($reader->getSheetIterator() as $sheet) { foreach ($sheet->getRowIterator() as $row) { $rowIndex++; $values = $row->toArray(); if (!$headerFound) { $normalizedRow = []; foreach ($values as $columnIndex => $cellValue) { $normalizedRow[$columnIndex] = $this->normalizeHeader($cellValue); } if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) { continue; } foreach ($normalizedRow as $columnIndex => $headerValue) { if ($headerValue === '' || $headerValue === 'Revenue Type') { continue; } $headers[] = $headerValue; $headerLookup[(int) $columnIndex] = $headerValue; } $headerRowIndex = $rowIndex; $headerFound = true; continue; } $mappedRow = []; foreach ($headerLookup as $columnIndex => $header) { $mappedRow[$header] = $this->normalizeCellValue($values[$columnIndex] ?? null); } if ($this->isBlankRow($mappedRow)) { continue; } $chunkRows[] = [ 'row_number' => $rowIndex, 'data' => $mappedRow, ]; $totalRows++; if (count($chunkRows) >= $chunkSize) { $chunkOffsets[] = ftell($rowsHandle); $this->writeChunkRow($rowsHandle, $chunkRows); $chunkRows = []; } } break; } } catch (Throwable $throwable) { throw new RuntimeException('Unable to build spreadsheet cache: ' . $throwable->getMessage(), 0, $throwable); } finally { $reader->close(); fclose($rowsHandle); } if (!$headerFound) { @unlink($rowsPath); throw new RuntimeException('Could not locate the header row in the uploaded workbook.'); } if ($chunkRows !== []) { $chunkOffsets[] = file_exists($rowsPath) ? filesize($rowsPath) : 0; $rowsHandle = fopen($rowsPath, 'ab'); if ($rowsHandle === false) { throw new RuntimeException('Unable to reopen the spreadsheet cache file.'); } try { $this->writeChunkRow($rowsHandle, $chunkRows); } finally { fclose($rowsHandle); } } $cache = [ 'file_path' => $filePath, 'headers' => $headers, 'header_lookup' => $headerLookup, 'header_row_index' => $headerRowIndex, 'total_rows' => $totalRows, 'chunk_size' => $chunkSize, 'chunk_count' => count($chunkOffsets), 'chunk_offsets' => $chunkOffsets, ]; if (file_put_contents( $metaPath, json_encode($cache, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE) ) === false) { throw new RuntimeException('Unable to write the spreadsheet cache metadata.'); } return $cache; } public function readChunkFromCache(string $cacheBasePath, int $chunkIndex): array { $cache = $this->readCacheMeta($cacheBasePath); $chunkOffsets = $cache['chunk_offsets'] ?? []; if (!is_array($chunkOffsets) || !array_key_exists($chunkIndex, $chunkOffsets)) { return [ 'rows' => [], 'cache' => $cache, ]; } $rowsPath = $this->cacheRowsPath($cacheBasePath); $rowsHandle = fopen($rowsPath, 'rb'); if ($rowsHandle === false) { throw new RuntimeException('Unable to open the spreadsheet cache file.'); } try { if (fseek($rowsHandle, (int) $chunkOffsets[$chunkIndex]) !== 0) { throw new RuntimeException('Unable to seek to the requested spreadsheet chunk.'); } $line = fgets($rowsHandle); if ($line === false) { throw new RuntimeException('Unable to read the requested spreadsheet chunk.'); } $rows = json_decode(trim($line), true, 512, JSON_THROW_ON_ERROR); if (!is_array($rows)) { throw new RuntimeException('Spreadsheet chunk data is invalid.'); } } finally { fclose($rowsHandle); } return [ 'rows' => $rows, 'cache' => $cache, ]; } public function deleteCache(string $cacheBasePath): void { @unlink($this->cacheMetaPath($cacheBasePath)); @unlink($this->cacheRowsPath($cacheBasePath)); } public function validateRequiredColumns(array $headers, array $requiredColumns): void { $missing = []; $headerMap = array_fill_keys($headers, true); foreach ($requiredColumns as $requiredColumn) { if (!isset($headerMap[$requiredColumn])) { $missing[] = $requiredColumn; } } if ($missing !== []) { throw new RuntimeException('Missing required column(s): ' . implode(', ', $missing)); } } private function normalizeHeader(mixed $value): string { if ($value === null) { return ''; } return trim((string) $value); } private function isHeaderRow(array $row, array $expectedHeaders): bool { $nonEmptyValues = array_values(array_filter($row, static fn (?string $value): bool => $value !== null && $value !== '')); if ($nonEmptyValues === []) { return false; } if ($expectedHeaders === []) { return count($nonEmptyValues) >= 2; } $lookup = array_fill_keys($nonEmptyValues, true); foreach ($expectedHeaders as $expectedHeader) { if (!isset($lookup[$expectedHeader])) { return false; } } return true; } private function loadWindow(string $filePath, int $startRow, int $endRow): array { try { $reader = IOFactory::createReaderForFile($filePath); $reader->setReadDataOnly(true); $sheetNames = $reader->listWorksheetNames($filePath); if ($sheetNames !== []) { $reader->setLoadSheetsOnly([$sheetNames[0]]); } $reader->setReadFilter(new ChunkReadFilter($startRow, $endRow)); $spreadsheet = $reader->load($filePath); } catch (Throwable $throwable) { throw new RuntimeException('Unable to read the uploaded Excel file: ' . $throwable->getMessage(), 0, $throwable); } try { $worksheet = $spreadsheet->getSheet(0); $rows = $worksheet->toArray(null, false, true, true); return [ 'rows' => $rows, 'highest_data_row' => $worksheet->getHighestDataRow(), ]; } finally { $spreadsheet->disconnectWorksheets(); unset($spreadsheet); } } private function createSpoutReader(string $filePath): SpoutReaderInterface { $extension = strtolower(pathinfo($filePath, PATHINFO_EXTENSION)); return match ($extension) { 'csv' => new CsvReader(), 'ods' => new OdsReader(), 'xlsx', 'xlsm' => new XlsxReader(), 'xls' => throw new RuntimeException('XLS files are not supported by the streaming reader. Please save the workbook as XLSX before importing.'), default => new XlsxReader(), }; } private function cacheMetaPath(string $cacheBasePath): string { return rtrim($cacheBasePath, '/\\') . '.meta.json'; } private function cacheRowsPath(string $cacheBasePath): string { return rtrim($cacheBasePath, '/\\') . '.rows.ndjson'; } private function readCacheMeta(string $cacheBasePath): array { $metaPath = $this->cacheMetaPath($cacheBasePath); if (!is_file($metaPath)) { throw new RuntimeException('Spreadsheet cache metadata is missing. Please restart the import.'); } $contents = file_get_contents($metaPath); if ($contents === false || $contents === '') { throw new RuntimeException('Spreadsheet cache metadata is unreadable. Please restart the import.'); } $cache = json_decode($contents, true, 512, JSON_THROW_ON_ERROR); if (!is_array($cache)) { throw new RuntimeException('Spreadsheet cache metadata is invalid. Please restart the import.'); } return $cache; } private function writeChunkRow($handle, array $chunkRows): void { $json = json_encode($chunkRows, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); if (fwrite($handle, $json . PHP_EOL) === false) { throw new RuntimeException('Unable to write spreadsheet cache chunk data.'); } } public function normalizeCellValue(mixed $value): ?string { if ($value === null) { return null; } if ($value instanceof \DateTimeInterface) { return $value->format('Y-m-d H:i:s'); } if (is_bool($value)) { return $value ? '1' : '0'; } if (is_int($value)) { return (string) $value; } if (is_float($value)) { if (fmod($value, 1.0) === 0.0) { return (string) (int) $value; } return rtrim(rtrim(sprintf('%.15F', $value), '0'), '.'); } $string = trim((string) $value); if ($string === '') { return null; } return $string; } private function isBlankRow(array $row): bool { foreach ($row as $value) { if ($value !== null && $value !== '') { return false; } } return true; } }