Initial
Takes in a file, process it and output a new file with additional columns.
This commit is contained in:
Stefan Lazic
@@ -0,0 +1,456 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
use OpenSpout\Reader\CSV\Reader as CsvReader;
|
||||
use OpenSpout\Reader\ODS\Reader as OdsReader;
|
||||
use OpenSpout\Reader\ReaderInterface as SpoutReaderInterface;
|
||||
use OpenSpout\Reader\XLSX\Reader as XlsxReader;
|
||||
use PhpOffice\PhpSpreadsheet\IOFactory;
|
||||
use RuntimeException;
|
||||
use Throwable;
|
||||
|
||||
final class SpreadsheetReader
|
||||
{
|
||||
public function detectHeader(string $filePath, array $expectedHeaders = [], int $scanChunkSize = 100, int $maxScanRows = 5000): array
|
||||
{
|
||||
for ($startRow = 1; $startRow <= $maxScanRows; $startRow += $scanChunkSize) {
|
||||
$endRow = min($maxScanRows, $startRow + $scanChunkSize - 1);
|
||||
$window = $this->loadWindow($filePath, $startRow, $endRow);
|
||||
|
||||
foreach ($window['rows'] as $rowIndex => $row) {
|
||||
$normalizedRow = [];
|
||||
foreach ($row as $columnLetter => $cellValue) {
|
||||
$normalizedRow[$columnLetter] = $this->normalizeHeader($cellValue);
|
||||
}
|
||||
|
||||
if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$headers = [];
|
||||
$headerLookup = [];
|
||||
foreach ($normalizedRow as $columnLetter => $headerValue) {
|
||||
if ($headerValue === '' || $headerValue === 'Revenue Type') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$headers[] = $headerValue;
|
||||
$headerLookup[$columnLetter] = $headerValue;
|
||||
}
|
||||
|
||||
return [
|
||||
'header_row_index' => (int) $rowIndex,
|
||||
'headers' => $headers,
|
||||
'header_lookup' => $headerLookup,
|
||||
'highest_data_row' => (int) $window['highest_data_row'],
|
||||
];
|
||||
}
|
||||
|
||||
if ((int) $window['highest_data_row'] < $endRow) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
throw new RuntimeException('Could not locate the header row in the uploaded workbook.');
|
||||
}
|
||||
|
||||
public function readChunk(string $filePath, int $startRow, int $chunkSize, array $headerLookup): array
|
||||
{
|
||||
$endRow = $startRow + $chunkSize - 1;
|
||||
$window = $this->loadWindow($filePath, $startRow, $endRow);
|
||||
$rows = [];
|
||||
|
||||
foreach ($window['rows'] as $rowIndex => $row) {
|
||||
$mappedRow = [];
|
||||
foreach ($headerLookup as $columnLetter => $header) {
|
||||
$mappedRow[$header] = $this->normalizeCellValue($row[$columnLetter] ?? null);
|
||||
}
|
||||
|
||||
if ($this->isBlankRow($mappedRow)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$rows[] = [
|
||||
'row_number' => (int) $rowIndex,
|
||||
'data' => $mappedRow,
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'rows' => $rows,
|
||||
'highest_data_row' => (int) $window['highest_data_row'],
|
||||
];
|
||||
}
|
||||
|
||||
public function read(string $filePath, array $expectedHeaders = []): array
|
||||
{
|
||||
$header = $this->detectHeader($filePath, $expectedHeaders);
|
||||
$allRows = [];
|
||||
$nextRow = $header['header_row_index'] + 1;
|
||||
|
||||
while ($nextRow <= $header['highest_data_row']) {
|
||||
$chunk = $this->readChunk($filePath, $nextRow, 500, $header['header_lookup']);
|
||||
$allRows = array_merge($allRows, $chunk['rows']);
|
||||
$nextRow += 500;
|
||||
}
|
||||
|
||||
return [
|
||||
'headers' => $header['headers'],
|
||||
'rows' => $allRows,
|
||||
];
|
||||
}
|
||||
|
||||
public function buildChunkCache(string $filePath, string $cacheBasePath, array $expectedHeaders = [], int $chunkSize = 50): array
|
||||
{
|
||||
if ($chunkSize < 1) {
|
||||
throw new RuntimeException('Chunk size must be at least 1.');
|
||||
}
|
||||
|
||||
$reader = $this->createSpoutReader($filePath);
|
||||
$metaPath = $this->cacheMetaPath($cacheBasePath);
|
||||
$rowsPath = $this->cacheRowsPath($cacheBasePath);
|
||||
$cacheDir = dirname($metaPath);
|
||||
if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) {
|
||||
throw new RuntimeException('Unable to create the spreadsheet cache directory.');
|
||||
}
|
||||
|
||||
$rowsHandle = fopen($rowsPath, 'wb');
|
||||
if ($rowsHandle === false) {
|
||||
throw new RuntimeException('Unable to create the spreadsheet cache file.');
|
||||
}
|
||||
|
||||
$headerFound = false;
|
||||
$rowIndex = 0;
|
||||
$chunkRows = [];
|
||||
$chunkOffsets = [];
|
||||
$totalRows = 0;
|
||||
$headerRowIndex = 0;
|
||||
$headers = [];
|
||||
$headerLookup = [];
|
||||
|
||||
try {
|
||||
$reader->open($filePath);
|
||||
|
||||
foreach ($reader->getSheetIterator() as $sheet) {
|
||||
foreach ($sheet->getRowIterator() as $row) {
|
||||
$rowIndex++;
|
||||
$values = $row->toArray();
|
||||
|
||||
if (!$headerFound) {
|
||||
$normalizedRow = [];
|
||||
foreach ($values as $columnIndex => $cellValue) {
|
||||
$normalizedRow[$columnIndex] = $this->normalizeHeader($cellValue);
|
||||
}
|
||||
|
||||
if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($normalizedRow as $columnIndex => $headerValue) {
|
||||
if ($headerValue === '' || $headerValue === 'Revenue Type') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$headers[] = $headerValue;
|
||||
$headerLookup[(int) $columnIndex] = $headerValue;
|
||||
}
|
||||
|
||||
$headerRowIndex = $rowIndex;
|
||||
$headerFound = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
$mappedRow = [];
|
||||
foreach ($headerLookup as $columnIndex => $header) {
|
||||
$mappedRow[$header] = $this->normalizeCellValue($values[$columnIndex] ?? null);
|
||||
}
|
||||
|
||||
if ($this->isBlankRow($mappedRow)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$chunkRows[] = [
|
||||
'row_number' => $rowIndex,
|
||||
'data' => $mappedRow,
|
||||
];
|
||||
$totalRows++;
|
||||
|
||||
if (count($chunkRows) >= $chunkSize) {
|
||||
$chunkOffsets[] = ftell($rowsHandle);
|
||||
$this->writeChunkRow($rowsHandle, $chunkRows);
|
||||
$chunkRows = [];
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
} catch (Throwable $throwable) {
|
||||
throw new RuntimeException('Unable to build spreadsheet cache: ' . $throwable->getMessage(), 0, $throwable);
|
||||
} finally {
|
||||
$reader->close();
|
||||
fclose($rowsHandle);
|
||||
}
|
||||
|
||||
if (!$headerFound) {
|
||||
@unlink($rowsPath);
|
||||
throw new RuntimeException('Could not locate the header row in the uploaded workbook.');
|
||||
}
|
||||
|
||||
if ($chunkRows !== []) {
|
||||
$chunkOffsets[] = file_exists($rowsPath) ? filesize($rowsPath) : 0;
|
||||
$rowsHandle = fopen($rowsPath, 'ab');
|
||||
if ($rowsHandle === false) {
|
||||
throw new RuntimeException('Unable to reopen the spreadsheet cache file.');
|
||||
}
|
||||
|
||||
try {
|
||||
$this->writeChunkRow($rowsHandle, $chunkRows);
|
||||
} finally {
|
||||
fclose($rowsHandle);
|
||||
}
|
||||
}
|
||||
|
||||
$cache = [
|
||||
'file_path' => $filePath,
|
||||
'headers' => $headers,
|
||||
'header_lookup' => $headerLookup,
|
||||
'header_row_index' => $headerRowIndex,
|
||||
'total_rows' => $totalRows,
|
||||
'chunk_size' => $chunkSize,
|
||||
'chunk_count' => count($chunkOffsets),
|
||||
'chunk_offsets' => $chunkOffsets,
|
||||
];
|
||||
|
||||
if (file_put_contents(
|
||||
$metaPath,
|
||||
json_encode($cache, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)
|
||||
) === false) {
|
||||
throw new RuntimeException('Unable to write the spreadsheet cache metadata.');
|
||||
}
|
||||
|
||||
return $cache;
|
||||
}
|
||||
|
||||
public function readChunkFromCache(string $cacheBasePath, int $chunkIndex): array
|
||||
{
|
||||
$cache = $this->readCacheMeta($cacheBasePath);
|
||||
$chunkOffsets = $cache['chunk_offsets'] ?? [];
|
||||
if (!is_array($chunkOffsets) || !array_key_exists($chunkIndex, $chunkOffsets)) {
|
||||
return [
|
||||
'rows' => [],
|
||||
'cache' => $cache,
|
||||
];
|
||||
}
|
||||
|
||||
$rowsPath = $this->cacheRowsPath($cacheBasePath);
|
||||
$rowsHandle = fopen($rowsPath, 'rb');
|
||||
if ($rowsHandle === false) {
|
||||
throw new RuntimeException('Unable to open the spreadsheet cache file.');
|
||||
}
|
||||
|
||||
try {
|
||||
if (fseek($rowsHandle, (int) $chunkOffsets[$chunkIndex]) !== 0) {
|
||||
throw new RuntimeException('Unable to seek to the requested spreadsheet chunk.');
|
||||
}
|
||||
|
||||
$line = fgets($rowsHandle);
|
||||
if ($line === false) {
|
||||
throw new RuntimeException('Unable to read the requested spreadsheet chunk.');
|
||||
}
|
||||
|
||||
$rows = json_decode(trim($line), true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($rows)) {
|
||||
throw new RuntimeException('Spreadsheet chunk data is invalid.');
|
||||
}
|
||||
} finally {
|
||||
fclose($rowsHandle);
|
||||
}
|
||||
|
||||
return [
|
||||
'rows' => $rows,
|
||||
'cache' => $cache,
|
||||
];
|
||||
}
|
||||
|
||||
public function deleteCache(string $cacheBasePath): void
|
||||
{
|
||||
@unlink($this->cacheMetaPath($cacheBasePath));
|
||||
@unlink($this->cacheRowsPath($cacheBasePath));
|
||||
}
|
||||
|
||||
public function validateRequiredColumns(array $headers, array $requiredColumns): void
|
||||
{
|
||||
$missing = [];
|
||||
$headerMap = array_fill_keys($headers, true);
|
||||
|
||||
foreach ($requiredColumns as $requiredColumn) {
|
||||
if (!isset($headerMap[$requiredColumn])) {
|
||||
$missing[] = $requiredColumn;
|
||||
}
|
||||
}
|
||||
|
||||
if ($missing !== []) {
|
||||
throw new RuntimeException('Missing required column(s): ' . implode(', ', $missing));
|
||||
}
|
||||
}
|
||||
|
||||
private function normalizeHeader(mixed $value): string
|
||||
{
|
||||
if ($value === null) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return trim((string) $value);
|
||||
}
|
||||
|
||||
private function isHeaderRow(array $row, array $expectedHeaders): bool
|
||||
{
|
||||
$nonEmptyValues = array_values(array_filter($row, static fn (?string $value): bool => $value !== null && $value !== ''));
|
||||
if ($nonEmptyValues === []) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($expectedHeaders === []) {
|
||||
return count($nonEmptyValues) >= 2;
|
||||
}
|
||||
|
||||
$lookup = array_fill_keys($nonEmptyValues, true);
|
||||
foreach ($expectedHeaders as $expectedHeader) {
|
||||
if (!isset($lookup[$expectedHeader])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function loadWindow(string $filePath, int $startRow, int $endRow): array
|
||||
{
|
||||
try {
|
||||
$reader = IOFactory::createReaderForFile($filePath);
|
||||
$reader->setReadDataOnly(true);
|
||||
|
||||
$sheetNames = $reader->listWorksheetNames($filePath);
|
||||
if ($sheetNames !== []) {
|
||||
$reader->setLoadSheetsOnly([$sheetNames[0]]);
|
||||
}
|
||||
|
||||
$reader->setReadFilter(new ChunkReadFilter($startRow, $endRow));
|
||||
$spreadsheet = $reader->load($filePath);
|
||||
} catch (Throwable $throwable) {
|
||||
throw new RuntimeException('Unable to read the uploaded Excel file: ' . $throwable->getMessage(), 0, $throwable);
|
||||
}
|
||||
|
||||
try {
|
||||
$worksheet = $spreadsheet->getSheet(0);
|
||||
$rows = $worksheet->toArray(null, false, true, true);
|
||||
|
||||
return [
|
||||
'rows' => $rows,
|
||||
'highest_data_row' => $worksheet->getHighestDataRow(),
|
||||
];
|
||||
} finally {
|
||||
$spreadsheet->disconnectWorksheets();
|
||||
unset($spreadsheet);
|
||||
}
|
||||
}
|
||||
|
||||
private function createSpoutReader(string $filePath): SpoutReaderInterface
|
||||
{
|
||||
$extension = strtolower(pathinfo($filePath, PATHINFO_EXTENSION));
|
||||
|
||||
return match ($extension) {
|
||||
'csv' => new CsvReader(),
|
||||
'ods' => new OdsReader(),
|
||||
'xlsx', 'xlsm' => new XlsxReader(),
|
||||
'xls' => throw new RuntimeException('XLS files are not supported by the streaming reader. Please save the workbook as XLSX before importing.'),
|
||||
default => new XlsxReader(),
|
||||
};
|
||||
}
|
||||
|
||||
private function cacheMetaPath(string $cacheBasePath): string
|
||||
{
|
||||
return rtrim($cacheBasePath, '/\\') . '.meta.json';
|
||||
}
|
||||
|
||||
private function cacheRowsPath(string $cacheBasePath): string
|
||||
{
|
||||
return rtrim($cacheBasePath, '/\\') . '.rows.ndjson';
|
||||
}
|
||||
|
||||
private function readCacheMeta(string $cacheBasePath): array
|
||||
{
|
||||
$metaPath = $this->cacheMetaPath($cacheBasePath);
|
||||
if (!is_file($metaPath)) {
|
||||
throw new RuntimeException('Spreadsheet cache metadata is missing. Please restart the import.');
|
||||
}
|
||||
|
||||
$contents = file_get_contents($metaPath);
|
||||
if ($contents === false || $contents === '') {
|
||||
throw new RuntimeException('Spreadsheet cache metadata is unreadable. Please restart the import.');
|
||||
}
|
||||
|
||||
$cache = json_decode($contents, true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($cache)) {
|
||||
throw new RuntimeException('Spreadsheet cache metadata is invalid. Please restart the import.');
|
||||
}
|
||||
|
||||
return $cache;
|
||||
}
|
||||
|
||||
private function writeChunkRow($handle, array $chunkRows): void
|
||||
{
|
||||
$json = json_encode($chunkRows, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
||||
if (fwrite($handle, $json . PHP_EOL) === false) {
|
||||
throw new RuntimeException('Unable to write spreadsheet cache chunk data.');
|
||||
}
|
||||
}
|
||||
|
||||
public function normalizeCellValue(mixed $value): ?string
|
||||
{
|
||||
if ($value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($value instanceof \DateTimeInterface) {
|
||||
return $value->format('Y-m-d H:i:s');
|
||||
}
|
||||
|
||||
if (is_bool($value)) {
|
||||
return $value ? '1' : '0';
|
||||
}
|
||||
|
||||
if (is_int($value)) {
|
||||
return (string) $value;
|
||||
}
|
||||
|
||||
if (is_float($value)) {
|
||||
if (fmod($value, 1.0) === 0.0) {
|
||||
return (string) (int) $value;
|
||||
}
|
||||
|
||||
return rtrim(rtrim(sprintf('%.15F', $value), '0'), '.');
|
||||
}
|
||||
|
||||
$string = trim((string) $value);
|
||||
if ($string === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $string;
|
||||
}
|
||||
|
||||
private function isBlankRow(array $row): bool
|
||||
{
|
||||
foreach ($row as $value) {
|
||||
if ($value !== null && $value !== '') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user