457 lines
15 KiB
PHP
457 lines
15 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Import;
|
|
|
|
use OpenSpout\Reader\CSV\Reader as CsvReader;
|
|
use OpenSpout\Reader\ODS\Reader as OdsReader;
|
|
use OpenSpout\Reader\ReaderInterface as SpoutReaderInterface;
|
|
use OpenSpout\Reader\XLSX\Reader as XlsxReader;
|
|
use PhpOffice\PhpSpreadsheet\IOFactory;
|
|
use RuntimeException;
|
|
use Throwable;
|
|
|
|
final class SpreadsheetReader
|
|
{
|
|
public function detectHeader(string $filePath, array $expectedHeaders = [], int $scanChunkSize = 100, int $maxScanRows = 5000): array
|
|
{
|
|
for ($startRow = 1; $startRow <= $maxScanRows; $startRow += $scanChunkSize) {
|
|
$endRow = min($maxScanRows, $startRow + $scanChunkSize - 1);
|
|
$window = $this->loadWindow($filePath, $startRow, $endRow);
|
|
|
|
foreach ($window['rows'] as $rowIndex => $row) {
|
|
$normalizedRow = [];
|
|
foreach ($row as $columnLetter => $cellValue) {
|
|
$normalizedRow[$columnLetter] = $this->normalizeHeader($cellValue);
|
|
}
|
|
|
|
if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) {
|
|
continue;
|
|
}
|
|
|
|
$headers = [];
|
|
$headerLookup = [];
|
|
foreach ($normalizedRow as $columnLetter => $headerValue) {
|
|
if ($headerValue === '' || $headerValue === 'Revenue Type') {
|
|
continue;
|
|
}
|
|
|
|
$headers[] = $headerValue;
|
|
$headerLookup[$columnLetter] = $headerValue;
|
|
}
|
|
|
|
return [
|
|
'header_row_index' => (int) $rowIndex,
|
|
'headers' => $headers,
|
|
'header_lookup' => $headerLookup,
|
|
'highest_data_row' => (int) $window['highest_data_row'],
|
|
];
|
|
}
|
|
|
|
if ((int) $window['highest_data_row'] < $endRow) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
throw new RuntimeException('Could not locate the header row in the uploaded workbook.');
|
|
}
|
|
|
|
public function readChunk(string $filePath, int $startRow, int $chunkSize, array $headerLookup): array
|
|
{
|
|
$endRow = $startRow + $chunkSize - 1;
|
|
$window = $this->loadWindow($filePath, $startRow, $endRow);
|
|
$rows = [];
|
|
|
|
foreach ($window['rows'] as $rowIndex => $row) {
|
|
$mappedRow = [];
|
|
foreach ($headerLookup as $columnLetter => $header) {
|
|
$mappedRow[$header] = $this->normalizeCellValue($row[$columnLetter] ?? null);
|
|
}
|
|
|
|
if ($this->isBlankRow($mappedRow)) {
|
|
continue;
|
|
}
|
|
|
|
$rows[] = [
|
|
'row_number' => (int) $rowIndex,
|
|
'data' => $mappedRow,
|
|
];
|
|
}
|
|
|
|
return [
|
|
'rows' => $rows,
|
|
'highest_data_row' => (int) $window['highest_data_row'],
|
|
];
|
|
}
|
|
|
|
public function read(string $filePath, array $expectedHeaders = []): array
|
|
{
|
|
$header = $this->detectHeader($filePath, $expectedHeaders);
|
|
$allRows = [];
|
|
$nextRow = $header['header_row_index'] + 1;
|
|
|
|
while ($nextRow <= $header['highest_data_row']) {
|
|
$chunk = $this->readChunk($filePath, $nextRow, 500, $header['header_lookup']);
|
|
$allRows = array_merge($allRows, $chunk['rows']);
|
|
$nextRow += 500;
|
|
}
|
|
|
|
return [
|
|
'headers' => $header['headers'],
|
|
'rows' => $allRows,
|
|
];
|
|
}
|
|
|
|
public function buildChunkCache(string $filePath, string $cacheBasePath, array $expectedHeaders = [], int $chunkSize = 50): array
|
|
{
|
|
if ($chunkSize < 1) {
|
|
throw new RuntimeException('Chunk size must be at least 1.');
|
|
}
|
|
|
|
$reader = $this->createSpoutReader($filePath);
|
|
$metaPath = $this->cacheMetaPath($cacheBasePath);
|
|
$rowsPath = $this->cacheRowsPath($cacheBasePath);
|
|
$cacheDir = dirname($metaPath);
|
|
if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) {
|
|
throw new RuntimeException('Unable to create the spreadsheet cache directory.');
|
|
}
|
|
|
|
$rowsHandle = fopen($rowsPath, 'wb');
|
|
if ($rowsHandle === false) {
|
|
throw new RuntimeException('Unable to create the spreadsheet cache file.');
|
|
}
|
|
|
|
$headerFound = false;
|
|
$rowIndex = 0;
|
|
$chunkRows = [];
|
|
$chunkOffsets = [];
|
|
$totalRows = 0;
|
|
$headerRowIndex = 0;
|
|
$headers = [];
|
|
$headerLookup = [];
|
|
|
|
try {
|
|
$reader->open($filePath);
|
|
|
|
foreach ($reader->getSheetIterator() as $sheet) {
|
|
foreach ($sheet->getRowIterator() as $row) {
|
|
$rowIndex++;
|
|
$values = $row->toArray();
|
|
|
|
if (!$headerFound) {
|
|
$normalizedRow = [];
|
|
foreach ($values as $columnIndex => $cellValue) {
|
|
$normalizedRow[$columnIndex] = $this->normalizeHeader($cellValue);
|
|
}
|
|
|
|
if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) {
|
|
continue;
|
|
}
|
|
|
|
foreach ($normalizedRow as $columnIndex => $headerValue) {
|
|
if ($headerValue === '' || $headerValue === 'Revenue Type') {
|
|
continue;
|
|
}
|
|
|
|
$headers[] = $headerValue;
|
|
$headerLookup[(int) $columnIndex] = $headerValue;
|
|
}
|
|
|
|
$headerRowIndex = $rowIndex;
|
|
$headerFound = true;
|
|
continue;
|
|
}
|
|
|
|
$mappedRow = [];
|
|
foreach ($headerLookup as $columnIndex => $header) {
|
|
$mappedRow[$header] = $this->normalizeCellValue($values[$columnIndex] ?? null);
|
|
}
|
|
|
|
if ($this->isBlankRow($mappedRow)) {
|
|
continue;
|
|
}
|
|
|
|
$chunkRows[] = [
|
|
'row_number' => $rowIndex,
|
|
'data' => $mappedRow,
|
|
];
|
|
$totalRows++;
|
|
|
|
if (count($chunkRows) >= $chunkSize) {
|
|
$chunkOffsets[] = ftell($rowsHandle);
|
|
$this->writeChunkRow($rowsHandle, $chunkRows);
|
|
$chunkRows = [];
|
|
}
|
|
}
|
|
|
|
break;
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
throw new RuntimeException('Unable to build spreadsheet cache: ' . $throwable->getMessage(), 0, $throwable);
|
|
} finally {
|
|
$reader->close();
|
|
fclose($rowsHandle);
|
|
}
|
|
|
|
if (!$headerFound) {
|
|
@unlink($rowsPath);
|
|
throw new RuntimeException('Could not locate the header row in the uploaded workbook.');
|
|
}
|
|
|
|
if ($chunkRows !== []) {
|
|
$chunkOffsets[] = file_exists($rowsPath) ? filesize($rowsPath) : 0;
|
|
$rowsHandle = fopen($rowsPath, 'ab');
|
|
if ($rowsHandle === false) {
|
|
throw new RuntimeException('Unable to reopen the spreadsheet cache file.');
|
|
}
|
|
|
|
try {
|
|
$this->writeChunkRow($rowsHandle, $chunkRows);
|
|
} finally {
|
|
fclose($rowsHandle);
|
|
}
|
|
}
|
|
|
|
$cache = [
|
|
'file_path' => $filePath,
|
|
'headers' => $headers,
|
|
'header_lookup' => $headerLookup,
|
|
'header_row_index' => $headerRowIndex,
|
|
'total_rows' => $totalRows,
|
|
'chunk_size' => $chunkSize,
|
|
'chunk_count' => count($chunkOffsets),
|
|
'chunk_offsets' => $chunkOffsets,
|
|
];
|
|
|
|
if (file_put_contents(
|
|
$metaPath,
|
|
json_encode($cache, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)
|
|
) === false) {
|
|
throw new RuntimeException('Unable to write the spreadsheet cache metadata.');
|
|
}
|
|
|
|
return $cache;
|
|
}
|
|
|
|
public function readChunkFromCache(string $cacheBasePath, int $chunkIndex): array
|
|
{
|
|
$cache = $this->readCacheMeta($cacheBasePath);
|
|
$chunkOffsets = $cache['chunk_offsets'] ?? [];
|
|
if (!is_array($chunkOffsets) || !array_key_exists($chunkIndex, $chunkOffsets)) {
|
|
return [
|
|
'rows' => [],
|
|
'cache' => $cache,
|
|
];
|
|
}
|
|
|
|
$rowsPath = $this->cacheRowsPath($cacheBasePath);
|
|
$rowsHandle = fopen($rowsPath, 'rb');
|
|
if ($rowsHandle === false) {
|
|
throw new RuntimeException('Unable to open the spreadsheet cache file.');
|
|
}
|
|
|
|
try {
|
|
if (fseek($rowsHandle, (int) $chunkOffsets[$chunkIndex]) !== 0) {
|
|
throw new RuntimeException('Unable to seek to the requested spreadsheet chunk.');
|
|
}
|
|
|
|
$line = fgets($rowsHandle);
|
|
if ($line === false) {
|
|
throw new RuntimeException('Unable to read the requested spreadsheet chunk.');
|
|
}
|
|
|
|
$rows = json_decode(trim($line), true, 512, JSON_THROW_ON_ERROR);
|
|
if (!is_array($rows)) {
|
|
throw new RuntimeException('Spreadsheet chunk data is invalid.');
|
|
}
|
|
} finally {
|
|
fclose($rowsHandle);
|
|
}
|
|
|
|
return [
|
|
'rows' => $rows,
|
|
'cache' => $cache,
|
|
];
|
|
}
|
|
|
|
public function deleteCache(string $cacheBasePath): void
|
|
{
|
|
@unlink($this->cacheMetaPath($cacheBasePath));
|
|
@unlink($this->cacheRowsPath($cacheBasePath));
|
|
}
|
|
|
|
public function validateRequiredColumns(array $headers, array $requiredColumns): void
|
|
{
|
|
$missing = [];
|
|
$headerMap = array_fill_keys($headers, true);
|
|
|
|
foreach ($requiredColumns as $requiredColumn) {
|
|
if (!isset($headerMap[$requiredColumn])) {
|
|
$missing[] = $requiredColumn;
|
|
}
|
|
}
|
|
|
|
if ($missing !== []) {
|
|
throw new RuntimeException('Missing required column(s): ' . implode(', ', $missing));
|
|
}
|
|
}
|
|
|
|
private function normalizeHeader(mixed $value): string
|
|
{
|
|
if ($value === null) {
|
|
return '';
|
|
}
|
|
|
|
return trim((string) $value);
|
|
}
|
|
|
|
private function isHeaderRow(array $row, array $expectedHeaders): bool
|
|
{
|
|
$nonEmptyValues = array_values(array_filter($row, static fn (?string $value): bool => $value !== null && $value !== ''));
|
|
if ($nonEmptyValues === []) {
|
|
return false;
|
|
}
|
|
|
|
if ($expectedHeaders === []) {
|
|
return count($nonEmptyValues) >= 2;
|
|
}
|
|
|
|
$lookup = array_fill_keys($nonEmptyValues, true);
|
|
foreach ($expectedHeaders as $expectedHeader) {
|
|
if (!isset($lookup[$expectedHeader])) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private function loadWindow(string $filePath, int $startRow, int $endRow): array
|
|
{
|
|
try {
|
|
$reader = IOFactory::createReaderForFile($filePath);
|
|
$reader->setReadDataOnly(true);
|
|
|
|
$sheetNames = $reader->listWorksheetNames($filePath);
|
|
if ($sheetNames !== []) {
|
|
$reader->setLoadSheetsOnly([$sheetNames[0]]);
|
|
}
|
|
|
|
$reader->setReadFilter(new ChunkReadFilter($startRow, $endRow));
|
|
$spreadsheet = $reader->load($filePath);
|
|
} catch (Throwable $throwable) {
|
|
throw new RuntimeException('Unable to read the uploaded Excel file: ' . $throwable->getMessage(), 0, $throwable);
|
|
}
|
|
|
|
try {
|
|
$worksheet = $spreadsheet->getSheet(0);
|
|
$rows = $worksheet->toArray(null, false, true, true);
|
|
|
|
return [
|
|
'rows' => $rows,
|
|
'highest_data_row' => $worksheet->getHighestDataRow(),
|
|
];
|
|
} finally {
|
|
$spreadsheet->disconnectWorksheets();
|
|
unset($spreadsheet);
|
|
}
|
|
}
|
|
|
|
private function createSpoutReader(string $filePath): SpoutReaderInterface
|
|
{
|
|
$extension = strtolower(pathinfo($filePath, PATHINFO_EXTENSION));
|
|
|
|
return match ($extension) {
|
|
'csv' => new CsvReader(),
|
|
'ods' => new OdsReader(),
|
|
'xlsx', 'xlsm' => new XlsxReader(),
|
|
'xls' => throw new RuntimeException('XLS files are not supported by the streaming reader. Please save the workbook as XLSX before importing.'),
|
|
default => new XlsxReader(),
|
|
};
|
|
}
|
|
|
|
private function cacheMetaPath(string $cacheBasePath): string
|
|
{
|
|
return rtrim($cacheBasePath, '/\\') . '.meta.json';
|
|
}
|
|
|
|
private function cacheRowsPath(string $cacheBasePath): string
|
|
{
|
|
return rtrim($cacheBasePath, '/\\') . '.rows.ndjson';
|
|
}
|
|
|
|
private function readCacheMeta(string $cacheBasePath): array
|
|
{
|
|
$metaPath = $this->cacheMetaPath($cacheBasePath);
|
|
if (!is_file($metaPath)) {
|
|
throw new RuntimeException('Spreadsheet cache metadata is missing. Please restart the import.');
|
|
}
|
|
|
|
$contents = file_get_contents($metaPath);
|
|
if ($contents === false || $contents === '') {
|
|
throw new RuntimeException('Spreadsheet cache metadata is unreadable. Please restart the import.');
|
|
}
|
|
|
|
$cache = json_decode($contents, true, 512, JSON_THROW_ON_ERROR);
|
|
if (!is_array($cache)) {
|
|
throw new RuntimeException('Spreadsheet cache metadata is invalid. Please restart the import.');
|
|
}
|
|
|
|
return $cache;
|
|
}
|
|
|
|
private function writeChunkRow($handle, array $chunkRows): void
|
|
{
|
|
$json = json_encode($chunkRows, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
|
if (fwrite($handle, $json . PHP_EOL) === false) {
|
|
throw new RuntimeException('Unable to write spreadsheet cache chunk data.');
|
|
}
|
|
}
|
|
|
|
public function normalizeCellValue(mixed $value): ?string
|
|
{
|
|
if ($value === null) {
|
|
return null;
|
|
}
|
|
|
|
if ($value instanceof \DateTimeInterface) {
|
|
return $value->format('Y-m-d H:i:s');
|
|
}
|
|
|
|
if (is_bool($value)) {
|
|
return $value ? '1' : '0';
|
|
}
|
|
|
|
if (is_int($value)) {
|
|
return (string) $value;
|
|
}
|
|
|
|
if (is_float($value)) {
|
|
if (fmod($value, 1.0) === 0.0) {
|
|
return (string) (int) $value;
|
|
}
|
|
|
|
return rtrim(rtrim(sprintf('%.15F', $value), '0'), '.');
|
|
}
|
|
|
|
$string = trim((string) $value);
|
|
if ($string === '') {
|
|
return null;
|
|
}
|
|
|
|
return $string;
|
|
}
|
|
|
|
private function isBlankRow(array $row): bool
|
|
{
|
|
foreach ($row as $value) {
|
|
if ($value !== null && $value !== '') {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
}
|