Initial
Takes in a file, process it and output a new file with additional columns.
This commit is contained in:
Stefan Lazic
@@ -0,0 +1,21 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
use PhpOffice\PhpSpreadsheet\Reader\IReadFilter;
|
||||
|
||||
final class ChunkReadFilter implements IReadFilter
|
||||
{
|
||||
public function __construct(
|
||||
private readonly int $startRow,
|
||||
private readonly int $endRow,
|
||||
) {
|
||||
}
|
||||
|
||||
public function readCell($columnAddress, $row, $worksheetName = ''): bool
|
||||
{
|
||||
return $row >= $this->startRow && $row <= $this->endRow;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
use RuntimeException;
|
||||
|
||||
final class ImportJobStore
|
||||
{
|
||||
public function __construct(
|
||||
private readonly string $storageDir = '',
|
||||
) {
|
||||
}
|
||||
|
||||
public function create(array $state): string
|
||||
{
|
||||
$token = bin2hex(random_bytes(16));
|
||||
$state['token'] = $token;
|
||||
$this->write($token, $state);
|
||||
|
||||
return $token;
|
||||
}
|
||||
|
||||
public function get(string $token): ?array
|
||||
{
|
||||
$path = $this->path($token);
|
||||
if (!is_file($path)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$contents = file_get_contents($path);
|
||||
if ($contents === false || $contents === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
$state = json_decode($contents, true);
|
||||
if (!is_array($state)) {
|
||||
throw new RuntimeException('The import job state is corrupt.');
|
||||
}
|
||||
|
||||
return $state;
|
||||
}
|
||||
|
||||
public function save(string $token, array $state): void
|
||||
{
|
||||
$state['token'] = $token;
|
||||
$this->write($token, $state);
|
||||
}
|
||||
|
||||
public function delete(string $token): void
|
||||
{
|
||||
$path = $this->path($token);
|
||||
if (is_file($path)) {
|
||||
@unlink($path);
|
||||
}
|
||||
}
|
||||
|
||||
public function pathForToken(string $token): string
|
||||
{
|
||||
return $this->path($token);
|
||||
}
|
||||
|
||||
private function write(string $token, array $state): void
|
||||
{
|
||||
$path = $this->path($token);
|
||||
$dir = dirname($path);
|
||||
if (!is_dir($dir) && !mkdir($dir, 0777, true) && !is_dir($dir)) {
|
||||
throw new RuntimeException('Unable to create the import job storage directory.');
|
||||
}
|
||||
|
||||
$json = json_encode($state, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
||||
$tmpPath = $path . '.' . bin2hex(random_bytes(6)) . '.tmp';
|
||||
if (file_put_contents($tmpPath, $json) === false) {
|
||||
throw new RuntimeException('Unable to write the import job state.');
|
||||
}
|
||||
|
||||
if (!@rename($tmpPath, $path)) {
|
||||
@unlink($tmpPath);
|
||||
throw new RuntimeException('Unable to persist the import job state.');
|
||||
}
|
||||
}
|
||||
|
||||
private function path(string $token): string
|
||||
{
|
||||
$safeToken = preg_replace('/[^a-zA-Z0-9_-]/', '', $token);
|
||||
if ($safeToken === '') {
|
||||
throw new RuntimeException('Invalid import job token.');
|
||||
}
|
||||
|
||||
$storageDir = $this->storageDir !== '' ? $this->storageDir : sys_get_temp_dir() . '/warner-import-jobs';
|
||||
|
||||
return rtrim($storageDir, '/\\') . '/' . $safeToken . '.json';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
use App\Database;
|
||||
use PDO;
|
||||
|
||||
final class ImportRepository
|
||||
{
|
||||
public function __construct(private readonly PDO $pdo)
|
||||
{
|
||||
}
|
||||
|
||||
public static function fromDatabase(Database $database): self
|
||||
{
|
||||
return new self($database->pdo());
|
||||
}
|
||||
|
||||
public function beginTransaction(): void
|
||||
{
|
||||
$this->pdo->beginTransaction();
|
||||
}
|
||||
|
||||
public function commit(): void
|
||||
{
|
||||
$this->pdo->commit();
|
||||
}
|
||||
|
||||
public function rollBack(): void
|
||||
{
|
||||
if ($this->pdo->inTransaction()) {
|
||||
$this->pdo->rollBack();
|
||||
}
|
||||
}
|
||||
|
||||
public function clearAll(): void
|
||||
{
|
||||
$this->pdo->exec('TRUNCATE TABLE import_rows, import_batches RESTART IDENTITY CASCADE');
|
||||
}
|
||||
|
||||
public function createBatch(string $sourceFilename, array $originalHeaders, array $calculatedHeaders, string $status = 'processing'): int
|
||||
{
|
||||
$statement = $this->pdo->prepare(
|
||||
'INSERT INTO import_batches (source_filename, original_headers, calculated_headers, status) VALUES (:source_filename, CAST(:original_headers AS jsonb), CAST(:calculated_headers AS jsonb), :status) RETURNING id'
|
||||
);
|
||||
$statement->execute([
|
||||
'source_filename' => $sourceFilename,
|
||||
'original_headers' => json_encode(array_values($originalHeaders), JSON_THROW_ON_ERROR),
|
||||
'calculated_headers' => json_encode(array_values($calculatedHeaders), JSON_THROW_ON_ERROR),
|
||||
'status' => $status,
|
||||
]);
|
||||
|
||||
return (int) $statement->fetchColumn();
|
||||
}
|
||||
|
||||
public function updateBatchSummary(int $batchId, int $rowCount, int $warningsCount, string $status = 'processing'): void
|
||||
{
|
||||
$statement = $this->pdo->prepare('UPDATE import_batches SET row_count = :row_count, warnings_count = :warnings_count, status = :status WHERE id = :id');
|
||||
$statement->execute([
|
||||
'row_count' => $rowCount,
|
||||
'warnings_count' => $warningsCount,
|
||||
'status' => $status,
|
||||
'id' => $batchId,
|
||||
]);
|
||||
}
|
||||
|
||||
public function insertRow(int $batchId, int $rowNumber, array $sourceRow, array $normalizedRow, array $calculatedRow, array $mergedRow): void
|
||||
{
|
||||
$statement = $this->pdo->prepare(
|
||||
'INSERT INTO import_rows (batch_id, row_number, source_row, normalized_row, calculated_row, merged_row) VALUES (:batch_id, :row_number, CAST(:source_row AS jsonb), CAST(:normalized_row AS jsonb), CAST(:calculated_row AS jsonb), CAST(:merged_row AS jsonb))'
|
||||
);
|
||||
$statement->execute([
|
||||
'batch_id' => $batchId,
|
||||
'row_number' => $rowNumber,
|
||||
'source_row' => json_encode($sourceRow, JSON_THROW_ON_ERROR),
|
||||
'normalized_row' => json_encode($normalizedRow, JSON_THROW_ON_ERROR),
|
||||
'calculated_row' => json_encode($calculatedRow, JSON_THROW_ON_ERROR),
|
||||
'merged_row' => json_encode($mergedRow, JSON_THROW_ON_ERROR),
|
||||
]);
|
||||
}
|
||||
|
||||
public function getLatestBatch(): ?array
|
||||
{
|
||||
$statement = $this->pdo->query("SELECT * FROM import_batches WHERE status = 'complete' ORDER BY id DESC LIMIT 1");
|
||||
$batch = $statement->fetch();
|
||||
|
||||
return $batch === false ? null : $this->decodeBatchRow($batch);
|
||||
}
|
||||
|
||||
public function countRowsForBatch(int $batchId): int
|
||||
{
|
||||
$statement = $this->pdo->prepare('SELECT COUNT(*) FROM import_rows WHERE batch_id = :batch_id');
|
||||
$statement->execute(['batch_id' => $batchId]);
|
||||
return (int) $statement->fetchColumn();
|
||||
}
|
||||
|
||||
public function getDistinctValuesForBatch(int $batchId, string $header, int $limit = 200): array
|
||||
{
|
||||
$statement = $this->pdo->prepare(
|
||||
"SELECT DISTINCT NULLIF(TRIM(COALESCE(jsonb_extract_path_text(merged_row, :header), '')), '') AS value
|
||||
FROM import_rows
|
||||
WHERE batch_id = :batch_id
|
||||
ORDER BY value ASC
|
||||
LIMIT :limit"
|
||||
);
|
||||
$statement->bindValue('batch_id', $batchId, PDO::PARAM_INT);
|
||||
$statement->bindValue('header', $header, PDO::PARAM_STR);
|
||||
$statement->bindValue('limit', $limit, PDO::PARAM_INT);
|
||||
$statement->execute();
|
||||
|
||||
$values = [];
|
||||
while ($row = $statement->fetch()) {
|
||||
$value = (string) ($row['value'] ?? '');
|
||||
if ($value !== '') {
|
||||
$values[] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
return $values;
|
||||
}
|
||||
|
||||
public function getDistinctValuesForBatchGrid(
|
||||
int $batchId,
|
||||
array $tableHeaders,
|
||||
string $columnKey,
|
||||
string $distinctSearch = '',
|
||||
int $limit = 500
|
||||
): array
|
||||
{
|
||||
if ($columnKey === 'row_number') {
|
||||
return [];
|
||||
}
|
||||
|
||||
$columnSql = $this->resolveGridColumnSql($tableHeaders, $columnKey);
|
||||
if ($columnSql === null) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$limit = max(1, min(500, $limit));
|
||||
$distinctSearch = trim($distinctSearch);
|
||||
$valueSql = 'COALESCE(' . $columnSql . ", '')";
|
||||
|
||||
$whereParts = ['batch_id = :batch_id'];
|
||||
$parameters = [
|
||||
'batch_id' => $batchId,
|
||||
];
|
||||
|
||||
if ($distinctSearch !== '') {
|
||||
$whereParts[] = $valueSql . ' ILIKE :distinct_search ESCAPE :distinct_search_esc';
|
||||
$parameters['distinct_search'] = '%' . $this->escapeLikePattern($distinctSearch) . '%';
|
||||
$parameters['distinct_search_esc'] = '\\';
|
||||
}
|
||||
|
||||
$statement = $this->pdo->prepare(
|
||||
'SELECT distinct_values.value
|
||||
FROM (
|
||||
SELECT DISTINCT ' . $valueSql . ' AS value
|
||||
FROM import_rows
|
||||
WHERE ' . implode(' AND ', $whereParts) . '
|
||||
) AS distinct_values
|
||||
ORDER BY LOWER(distinct_values.value) ASC, distinct_values.value ASC
|
||||
LIMIT :limit'
|
||||
);
|
||||
|
||||
foreach ($parameters as $name => $value) {
|
||||
$statement->bindValue(':' . $name, $value, $name === 'batch_id' ? PDO::PARAM_INT : PDO::PARAM_STR);
|
||||
}
|
||||
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
|
||||
$statement->execute();
|
||||
|
||||
$values = [];
|
||||
while ($row = $statement->fetch()) {
|
||||
$value = (string) ($row['value'] ?? '');
|
||||
if (!in_array($value, $values, true)) {
|
||||
$values[] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
return $values;
|
||||
}
|
||||
|
||||
public function paginateRows(int $batchId, int $limit, int $offset, string $sortDirection = 'asc'): array
|
||||
{
|
||||
return $this->paginateRowsFiltered($batchId, $limit, $offset, '', [], [], null, $sortDirection);
|
||||
}
|
||||
|
||||
public function countRowsForBatchFiltered(
|
||||
int $batchId,
|
||||
string $query = '',
|
||||
array $tableHeaders = [],
|
||||
array $columnSearches = []
|
||||
): int
|
||||
{
|
||||
[$whereClause, $parameters] = $this->buildDataTableClause($tableHeaders, $query, $columnSearches);
|
||||
$statement = $this->pdo->prepare(
|
||||
'SELECT COUNT(*)
|
||||
FROM import_rows
|
||||
WHERE batch_id = :batch_id' . $whereClause
|
||||
);
|
||||
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
|
||||
foreach ($parameters as $name => $value) {
|
||||
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
|
||||
}
|
||||
$statement->execute();
|
||||
|
||||
return (int) $statement->fetchColumn();
|
||||
}
|
||||
|
||||
public function countRowsForBatchGrid(
|
||||
int $batchId,
|
||||
string $query = '',
|
||||
array $tableHeaders = [],
|
||||
array $filterModel = []
|
||||
): int
|
||||
{
|
||||
[$whereClause, $parameters] = $this->buildGridWhereClause($tableHeaders, $query, $filterModel);
|
||||
$statement = $this->pdo->prepare(
|
||||
'SELECT COUNT(*)
|
||||
FROM import_rows
|
||||
WHERE batch_id = :batch_id' . $whereClause
|
||||
);
|
||||
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
|
||||
foreach ($parameters as $name => $value) {
|
||||
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
|
||||
}
|
||||
$statement->execute();
|
||||
|
||||
return (int) $statement->fetchColumn();
|
||||
}
|
||||
|
||||
public function paginateRowsFiltered(
|
||||
int $batchId,
|
||||
int $limit,
|
||||
int $offset,
|
||||
string $query = '',
|
||||
array $tableHeaders = [],
|
||||
array $columnSearches = [],
|
||||
?int $orderColumn = null,
|
||||
string $sortDirection = 'asc'
|
||||
): array
|
||||
{
|
||||
[$whereClause, $parameters] = $this->buildDataTableClause($tableHeaders, $query, $columnSearches);
|
||||
[$orderClause, $orderParameters] = $this->buildOrderClause($tableHeaders, $orderColumn, $sortDirection);
|
||||
$statement = $this->pdo->prepare(
|
||||
'SELECT row_number, source_row, normalized_row, calculated_row, merged_row
|
||||
FROM import_rows
|
||||
WHERE batch_id = :batch_id' . $whereClause . '
|
||||
' . $orderClause . '
|
||||
LIMIT :limit OFFSET :offset'
|
||||
);
|
||||
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
|
||||
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
|
||||
$statement->bindValue(':offset', $offset, PDO::PARAM_INT);
|
||||
foreach ($parameters as $name => $value) {
|
||||
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
|
||||
}
|
||||
foreach ($orderParameters as $name => $value) {
|
||||
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
|
||||
}
|
||||
$statement->execute();
|
||||
|
||||
$rows = [];
|
||||
while ($row = $statement->fetch()) {
|
||||
$rows[] = $this->decodeRowPayload($row);
|
||||
}
|
||||
|
||||
return $rows;
|
||||
}
|
||||
|
||||
public function paginateRowsGrid(
|
||||
int $batchId,
|
||||
int $limit,
|
||||
int $offset,
|
||||
string $query = '',
|
||||
array $tableHeaders = [],
|
||||
array $filterModel = [],
|
||||
array $sortModel = []
|
||||
): array
|
||||
{
|
||||
[$whereClause, $parameters] = $this->buildGridWhereClause($tableHeaders, $query, $filterModel);
|
||||
[$orderClause, $orderParameters] = $this->buildGridOrderClause($tableHeaders, $sortModel);
|
||||
$statement = $this->pdo->prepare(
|
||||
'SELECT row_number, merged_row
|
||||
FROM import_rows
|
||||
WHERE batch_id = :batch_id' . $whereClause . '
|
||||
' . $orderClause . '
|
||||
LIMIT :limit OFFSET :offset'
|
||||
);
|
||||
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
|
||||
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
|
||||
$statement->bindValue(':offset', $offset, PDO::PARAM_INT);
|
||||
foreach ($parameters as $name => $value) {
|
||||
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
|
||||
}
|
||||
foreach ($orderParameters as $name => $value) {
|
||||
$statement->bindValue(':' . $name, $value, PDO::PARAM_STR);
|
||||
}
|
||||
$statement->execute();
|
||||
|
||||
$rows = [];
|
||||
while ($row = $statement->fetch()) {
|
||||
$rows[] = $this->decodeGridRowPayload($row);
|
||||
}
|
||||
|
||||
return $rows;
|
||||
}
|
||||
|
||||
public function getAllRowsCount(): int
|
||||
{
|
||||
return (int) $this->pdo->query('SELECT COUNT(*) FROM import_rows')->fetchColumn();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array<string, mixed>>
|
||||
*/
|
||||
public function exportRowsForBatch(int $batchId, int $limit, int $offset): array
|
||||
{
|
||||
$statement = $this->pdo->prepare(
|
||||
'SELECT row_number, merged_row
|
||||
FROM import_rows
|
||||
WHERE batch_id = :batch_id
|
||||
ORDER BY row_number ASC
|
||||
LIMIT :limit OFFSET :offset'
|
||||
);
|
||||
$statement->bindValue(':batch_id', $batchId, PDO::PARAM_INT);
|
||||
$statement->bindValue(':limit', $limit, PDO::PARAM_INT);
|
||||
$statement->bindValue(':offset', $offset, PDO::PARAM_INT);
|
||||
$statement->execute();
|
||||
|
||||
$rows = [];
|
||||
while ($row = $statement->fetch()) {
|
||||
$rows[] = $this->decodeGridRowPayload($row);
|
||||
}
|
||||
|
||||
return $rows;
|
||||
}
|
||||
|
||||
public function pruneExceptBatch(int $batchId): void
|
||||
{
|
||||
$statement = $this->pdo->prepare('DELETE FROM import_batches WHERE id <> :batch_id');
|
||||
$statement->execute(['batch_id' => $batchId]);
|
||||
}
|
||||
|
||||
private function decodeBatchRow(array $row): array
|
||||
{
|
||||
$row['original_headers'] = json_decode((string) $row['original_headers'], true, 512, JSON_THROW_ON_ERROR);
|
||||
$row['calculated_headers'] = json_decode((string) $row['calculated_headers'], true, 512, JSON_THROW_ON_ERROR);
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
private function decodeRowPayload(array $row): array
|
||||
{
|
||||
$row['source_row'] = json_decode((string) $row['source_row'], true, 512, JSON_THROW_ON_ERROR);
|
||||
$row['normalized_row'] = json_decode((string) $row['normalized_row'], true, 512, JSON_THROW_ON_ERROR);
|
||||
$row['calculated_row'] = json_decode((string) $row['calculated_row'], true, 512, JSON_THROW_ON_ERROR);
|
||||
$row['merged_row'] = json_decode((string) $row['merged_row'], true, 512, JSON_THROW_ON_ERROR);
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
private function decodeGridRowPayload(array $row): array
|
||||
{
|
||||
$mergedRow = json_decode((string) $row['merged_row'], true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($mergedRow)) {
|
||||
$mergedRow = [];
|
||||
}
|
||||
|
||||
return array_merge($mergedRow, [
|
||||
'row_number' => (int) $row['row_number'],
|
||||
]);
|
||||
}
|
||||
|
||||
private function resolveGridColumnSql(array $tableHeaders, string $columnKey): ?string
|
||||
{
|
||||
if ($columnKey === 'row_number') {
|
||||
return 'row_number';
|
||||
}
|
||||
|
||||
$headerIndex = array_search($columnKey, $tableHeaders, true);
|
||||
if ($headerIndex === false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->columnSqlForHeader((string) $tableHeaders[(int) $headerIndex]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:string,1:array<string,string>}
|
||||
*/
|
||||
private function buildGridWhereClause(array $tableHeaders, string $globalSearch, array $filterModel): array
|
||||
{
|
||||
$conditions = [];
|
||||
$parameters = [];
|
||||
|
||||
$globalSearch = trim($globalSearch);
|
||||
if ($globalSearch !== '') {
|
||||
$globalQueryRow = 'grid_global_query_row';
|
||||
$globalQueryMerged = 'grid_global_query_merged';
|
||||
$conditions[] = "(
|
||||
CAST(row_number AS TEXT) ILIKE :{$globalQueryRow} ESCAPE :{$globalQueryRow}_esc
|
||||
OR CAST(merged_row AS TEXT) ILIKE :{$globalQueryMerged} ESCAPE :{$globalQueryMerged}_esc
|
||||
)";
|
||||
$escapedGlobalSearch = '%' . $this->escapeLikePattern($globalSearch) . '%';
|
||||
$parameters[$globalQueryRow] = $escapedGlobalSearch;
|
||||
$parameters[$globalQueryRow . '_esc'] = '\\';
|
||||
$parameters[$globalQueryMerged] = $escapedGlobalSearch;
|
||||
$parameters[$globalQueryMerged . '_esc'] = '\\';
|
||||
}
|
||||
|
||||
foreach ($filterModel as $columnId => $model) {
|
||||
if (!is_array($model)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$columnKey = (string) $columnId;
|
||||
$columnSql = $this->resolveGridColumnSql($tableHeaders, $columnKey);
|
||||
if ($columnSql === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$counter = 0;
|
||||
$clause = $this->buildGridFilterClause($columnSql, $model, 'grid_' . preg_replace('/[^a-zA-Z0-9_]+/', '_', $columnKey), $parameters, $counter);
|
||||
if ($clause !== '') {
|
||||
$conditions[] = $clause;
|
||||
}
|
||||
}
|
||||
|
||||
if ($conditions === []) {
|
||||
return ['', []];
|
||||
}
|
||||
|
||||
return [
|
||||
' AND (' . implode(' AND ', $conditions) . ')',
|
||||
$parameters,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:string,1:array<string,string>}
|
||||
*/
|
||||
private function buildGridOrderClause(array $tableHeaders, array $sortModel): array
|
||||
{
|
||||
$orderParts = [];
|
||||
$parameters = [];
|
||||
$hasRowNumberSort = false;
|
||||
|
||||
foreach ($sortModel as $index => $sortItem) {
|
||||
if (!is_array($sortItem)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$columnKey = (string) ($sortItem['colId'] ?? '');
|
||||
$sortDirection = strtolower((string) ($sortItem['sort'] ?? 'asc')) === 'desc' ? 'DESC' : 'ASC';
|
||||
|
||||
$columnSql = $this->resolveGridColumnSql($tableHeaders, $columnKey);
|
||||
if ($columnSql === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($columnKey === 'row_number') {
|
||||
$orderParts[] = 'row_number ' . $sortDirection;
|
||||
$hasRowNumberSort = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
$orderParts[] = 'LOWER(COALESCE(' . $columnSql . ", '')) " . $sortDirection;
|
||||
}
|
||||
|
||||
if ($orderParts === []) {
|
||||
return ['ORDER BY row_number ASC', []];
|
||||
}
|
||||
|
||||
if (!$hasRowNumberSort) {
|
||||
$orderParts[] = 'row_number ASC';
|
||||
}
|
||||
|
||||
return ['ORDER BY ' . implode(', ', $orderParts), $parameters];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string,string>
|
||||
*/
|
||||
private function buildGridFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
|
||||
{
|
||||
if (isset($model['operator'], $model['condition1'], $model['condition2']) && is_array($model['condition1']) && is_array($model['condition2'])) {
|
||||
$left = $this->buildGridFilterClause($columnSql, $model['condition1'], $paramPrefix . '_c1', $parameters, $counter);
|
||||
$right = $this->buildGridFilterClause($columnSql, $model['condition2'], $paramPrefix . '_c2', $parameters, $counter);
|
||||
$operator = strtoupper((string) $model['operator']) === 'OR' ? 'OR' : 'AND';
|
||||
return '(' . $left . ' ' . $operator . ' ' . $right . ')';
|
||||
}
|
||||
|
||||
$filterType = strtolower((string) ($model['filterType'] ?? 'text'));
|
||||
if ($filterType === 'set') {
|
||||
return $this->buildGridSetFilterClause($columnSql, $model, $paramPrefix, $parameters, $counter);
|
||||
}
|
||||
|
||||
if ($filterType === 'number') {
|
||||
return $this->buildGridNumberFilterClause($columnSql, $model, $paramPrefix, $parameters, $counter);
|
||||
}
|
||||
|
||||
return $this->buildGridTextFilterClause($columnSql, $model, $paramPrefix, $parameters, $counter);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string,string>
|
||||
*/
|
||||
private function buildGridTextFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
|
||||
{
|
||||
$type = strtolower((string) ($model['type'] ?? 'contains'));
|
||||
$filter = trim((string) ($model['filter'] ?? ''));
|
||||
|
||||
$paramName = $paramPrefix . '_' . $counter++;
|
||||
$escapedFilter = $this->escapeLikePattern($filter);
|
||||
|
||||
return match ($type) {
|
||||
'blank' => 'COALESCE(TRIM(' . $columnSql . "), '') = ''",
|
||||
'notblank' => 'COALESCE(TRIM(' . $columnSql . "), '') <> ''",
|
||||
'equals' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true),
|
||||
'notequal' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, false),
|
||||
'startswith' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true, false, true),
|
||||
'endswith' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true, true, false),
|
||||
'notcontains' => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, false, true, true),
|
||||
default => $this->addTextFilterParameter($columnSql, $paramName, $escapedFilter, $parameters, true, true, true),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string,string>
|
||||
*/
|
||||
private function buildGridNumberFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
|
||||
{
|
||||
$type = strtolower((string) ($model['type'] ?? 'equals'));
|
||||
$filter = $model['filter'] ?? null;
|
||||
$filterTo = $model['filterTo'] ?? null;
|
||||
|
||||
$paramName = $paramPrefix . '_' . $counter++;
|
||||
$from = is_numeric($filter) ? (string) $filter : '';
|
||||
$to = is_numeric($filterTo) ? (string) $filterTo : '';
|
||||
|
||||
return match ($type) {
|
||||
'blank' => '1 = 0',
|
||||
'notblank' => '1 = 1',
|
||||
'notequal' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '<>'),
|
||||
'lessthan' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '<'),
|
||||
'lessthanorequal' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '<='),
|
||||
'greaterthan' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '>'),
|
||||
'greaterthanorequal' => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '>='),
|
||||
'inrange' => $this->addNumericRangeFilterParameter($columnSql, $paramName, $from, $to, $parameters),
|
||||
default => $this->addNumericFilterParameter($columnSql, $paramName, $from, $parameters, '='),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string,string>
|
||||
*/
|
||||
private function buildGridSetFilterClause(string $columnSql, array $model, string $paramPrefix, array &$parameters, int &$counter): string
|
||||
{
|
||||
$values = $model['values'] ?? [];
|
||||
if (!is_array($values) || $values === []) {
|
||||
return '1 = 1';
|
||||
}
|
||||
|
||||
$placeholders = [];
|
||||
foreach ($values as $value) {
|
||||
if (!is_scalar($value) && $value !== null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$paramName = $paramPrefix . '_' . $counter++;
|
||||
$parameters[$paramName] = (string) $value;
|
||||
$placeholders[] = ':' . $paramName;
|
||||
}
|
||||
|
||||
if ($placeholders === []) {
|
||||
return '1 = 1';
|
||||
}
|
||||
|
||||
return $columnSql . ' IN (' . implode(', ', $placeholders) . ')';
|
||||
}
|
||||
|
||||
private function addTextFilterParameter(
|
||||
string $columnSql,
|
||||
string $paramName,
|
||||
string $escapedFilter,
|
||||
array &$parameters,
|
||||
bool $positive = true,
|
||||
bool $prependWildcard = false,
|
||||
bool $appendWildcard = false
|
||||
): string
|
||||
{
|
||||
if ($escapedFilter === '') {
|
||||
return '1 = 1';
|
||||
}
|
||||
|
||||
$parameters[$paramName] = ($prependWildcard ? '%' : '') . $escapedFilter . ($appendWildcard ? '%' : '');
|
||||
$parameters[$paramName . '_esc'] = '\\';
|
||||
$clause = $columnSql . " ILIKE :{$paramName} ESCAPE :{$paramName}_esc";
|
||||
return $positive ? $clause : 'NOT (' . $clause . ')';
|
||||
}
|
||||
|
||||
private function addNumericFilterParameter(string $columnSql, string $paramName, string $value, array &$parameters, string $operator): string
|
||||
{
|
||||
if ($value === '') {
|
||||
return '1 = 1';
|
||||
}
|
||||
|
||||
$parameters[$paramName] = $value;
|
||||
return $columnSql . " {$operator} :{$paramName}";
|
||||
}
|
||||
|
||||
private function addNumericRangeFilterParameter(string $columnSql, string $paramName, string $from, string $to, array &$parameters): string
|
||||
{
|
||||
if ($from === '' || $to === '') {
|
||||
return '1 = 1';
|
||||
}
|
||||
|
||||
$fromName = $paramName . '_from';
|
||||
$toName = $paramName . '_to';
|
||||
$parameters[$fromName] = $from;
|
||||
$parameters[$toName] = $to;
|
||||
|
||||
return $columnSql . " BETWEEN :{$fromName} AND :{$toName}";
|
||||
}
|
||||
|
||||
private function columnSqlForHeader(string $header): string
|
||||
{
|
||||
return 'COALESCE(merged_row ->> ' . $this->pdo->quote($header) . ", '')";
|
||||
}
|
||||
|
||||
private function escapeLikePattern(string $value): string
|
||||
{
|
||||
return str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $value);
|
||||
}
|
||||
|
||||
private function isAssocArray(array $value): bool
|
||||
{
|
||||
return array_keys($value) !== range(0, count($value) - 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:string,1:array<string,string>}
|
||||
*/
|
||||
private function buildDataTableClause(array $tableHeaders, string $globalSearch, array $columnSearches): array
|
||||
{
|
||||
$conditions = [];
|
||||
$parameters = [];
|
||||
|
||||
$globalSearch = trim($globalSearch);
|
||||
if ($globalSearch !== '') {
|
||||
$conditions[] = "(
|
||||
CAST(row_number AS TEXT) ILIKE :global_query ESCAPE :global_query_esc
|
||||
OR CAST(merged_row AS TEXT) ILIKE :global_query ESCAPE :global_query_esc
|
||||
)";
|
||||
$parameters['global_query'] = '%' . str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $globalSearch) . '%';
|
||||
$parameters['global_query_esc'] = '\\';
|
||||
}
|
||||
|
||||
foreach ($columnSearches as $index => $search) {
|
||||
$search = trim((string) $search);
|
||||
if ($search === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$parameterName = 'column_query_' . (int) $index;
|
||||
if ((int) $index === 0) {
|
||||
$conditions[] = "CAST(row_number AS TEXT) = :{$parameterName}";
|
||||
$parameters[$parameterName] = $search;
|
||||
continue;
|
||||
}
|
||||
|
||||
$headerIndex = (int) $index - 1;
|
||||
if (!isset($tableHeaders[$headerIndex])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$headerParameter = 'column_header_' . (int) $index;
|
||||
$conditions[] = "COALESCE(jsonb_extract_path_text(merged_row, :{$headerParameter}), '') = :{$parameterName}";
|
||||
$parameters[$headerParameter] = (string) $tableHeaders[$headerIndex];
|
||||
$parameters[$parameterName] = $search;
|
||||
}
|
||||
|
||||
if ($conditions === []) {
|
||||
return ['', []];
|
||||
}
|
||||
|
||||
return [
|
||||
' AND (' . implode(' AND ', $conditions) . ')',
|
||||
$parameters,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:string,1:array<string,string>}
|
||||
*/
|
||||
private function buildOrderClause(array $tableHeaders, ?int $orderColumn, string $sortDirection): array
|
||||
{
|
||||
$sortDirection = strtoupper($sortDirection) === 'DESC' ? 'DESC' : 'ASC';
|
||||
|
||||
if ($orderColumn === null || $orderColumn < 0) {
|
||||
return ['ORDER BY row_number DESC', []];
|
||||
}
|
||||
|
||||
if ($orderColumn === 0) {
|
||||
return ['ORDER BY row_number ' . $sortDirection, []];
|
||||
}
|
||||
|
||||
$headerIndex = $orderColumn - 1;
|
||||
if (!isset($tableHeaders[$headerIndex])) {
|
||||
return ['ORDER BY row_number DESC', []];
|
||||
}
|
||||
|
||||
return [
|
||||
'ORDER BY LOWER(COALESCE(jsonb_extract_path_text(merged_row, :order_header), \'\')) ' . $sortDirection . ', row_number ASC',
|
||||
[
|
||||
'order_header' => (string) $tableHeaders[$headerIndex],
|
||||
],
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
final class ImportResult
|
||||
{
|
||||
public function __construct(
|
||||
public readonly ?int $batchId,
|
||||
public readonly int $importedRows,
|
||||
public readonly int $warningsCount,
|
||||
public readonly array $warnings,
|
||||
public readonly array $originalHeaders,
|
||||
public readonly array $calculatedHeaders,
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
use RuntimeException;
|
||||
use Throwable;
|
||||
|
||||
final class ImportService
|
||||
{
|
||||
private const REQUIRED_COLUMNS = [
|
||||
'Accrual Category',
|
||||
'Local Selling Company (S/T)',
|
||||
'Contracting Party',
|
||||
'Rights Holder Liable Flag',
|
||||
'Global Transfer Pricing Flag',
|
||||
'Domestic/ Foreign Flag',
|
||||
'Ownership Type',
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
private readonly SpreadsheetReader $reader,
|
||||
private readonly SpreadsheetRowProcessor $processor,
|
||||
private readonly ImportRepository $repository,
|
||||
private readonly int $chunkSize = 5,
|
||||
) {
|
||||
}
|
||||
|
||||
public function requiredColumns(): array
|
||||
{
|
||||
return self::REQUIRED_COLUMNS;
|
||||
}
|
||||
|
||||
public function import(string $filePath, string $sourceFilename): ImportResult
|
||||
{
|
||||
$state = [
|
||||
'filePath' => $filePath,
|
||||
'sourceFilename' => $sourceFilename,
|
||||
'sourceExtension' => strtolower(pathinfo($filePath, PATHINFO_EXTENSION)),
|
||||
'status' => 'pending',
|
||||
'importedRows' => 0,
|
||||
'warningsCount' => 0,
|
||||
'warnings' => [],
|
||||
];
|
||||
|
||||
while (!in_array(($state['status'] ?? 'pending'), ['completed', 'error'], true)) {
|
||||
$state = $this->processJobState($state, $this->chunkSize);
|
||||
}
|
||||
|
||||
if (($state['status'] ?? '') === 'error') {
|
||||
throw new RuntimeException('Import failed: ' . ($state['error'] ?? 'Unknown error'));
|
||||
}
|
||||
|
||||
return new ImportResult(
|
||||
isset($state['batchId']) ? (int) $state['batchId'] : null,
|
||||
(int) ($state['importedRows'] ?? 0),
|
||||
(int) ($state['warningsCount'] ?? 0),
|
||||
is_array($state['warnings'] ?? null) ? array_values($state['warnings']) : [],
|
||||
is_array($state['headers'] ?? null) ? array_values($state['headers']) : [],
|
||||
$this->processor->calculatedHeaders()
|
||||
);
|
||||
}
|
||||
|
||||
public function processJobState(array $state, ?int $chunkSize = null): array
|
||||
{
|
||||
$chunkSize = max(1, $chunkSize ?? $this->chunkSize);
|
||||
$extension = strtolower((string) ($state['sourceExtension'] ?? pathinfo((string) ($state['filePath'] ?? ''), PATHINFO_EXTENSION)));
|
||||
if ($extension === 'xls') {
|
||||
return $this->processLegacyJobState($state, $chunkSize);
|
||||
}
|
||||
|
||||
return $this->processStreamingJobState($state, $chunkSize);
|
||||
}
|
||||
|
||||
private function processStreamingJobState(array $state, int $chunkSize): array
|
||||
{
|
||||
if (($state['status'] ?? 'pending') === 'completed' || ($state['status'] ?? 'pending') === 'error') {
|
||||
return $state;
|
||||
}
|
||||
|
||||
try {
|
||||
$state['importedRows'] = (int) ($state['importedRows'] ?? 0);
|
||||
$state['warningsCount'] = (int) ($state['warningsCount'] ?? 0);
|
||||
$state['warnings'] = is_array($state['warnings'] ?? null) ? array_values($state['warnings']) : [];
|
||||
|
||||
if (!isset($state['cacheBasePath'])) {
|
||||
$state['cacheBasePath'] = $this->cacheBasePath($state);
|
||||
$cache = $this->reader->buildChunkCache(
|
||||
(string) $state['filePath'],
|
||||
(string) $state['cacheBasePath'],
|
||||
self::REQUIRED_COLUMNS,
|
||||
$chunkSize
|
||||
);
|
||||
$this->reader->validateRequiredColumns($cache['headers'], self::REQUIRED_COLUMNS);
|
||||
|
||||
$state['headers'] = $cache['headers'];
|
||||
$state['headerLookup'] = $cache['header_lookup'];
|
||||
$state['headerRowIndex'] = (int) $cache['header_row_index'];
|
||||
$state['totalRows'] = (int) $cache['total_rows'];
|
||||
$state['chunkCount'] = (int) $cache['chunk_count'];
|
||||
$state['chunkSize'] = (int) $cache['chunk_size'];
|
||||
$state['nextChunkIndex'] = (int) ($state['nextChunkIndex'] ?? 0);
|
||||
$state['status'] = 'cached';
|
||||
$state['done'] = false;
|
||||
$state['message'] = 'Workbook cached and ready for chunked processing.';
|
||||
|
||||
return $state;
|
||||
}
|
||||
|
||||
if (!isset($state['batchId'])) {
|
||||
$state['batchId'] = $this->repository->createBatch(
|
||||
(string) $state['sourceFilename'],
|
||||
is_array($state['headers'] ?? null) ? $state['headers'] : [],
|
||||
$this->processor->calculatedHeaders(),
|
||||
'processing'
|
||||
);
|
||||
}
|
||||
|
||||
$chunkCount = (int) ($state['chunkCount'] ?? 0);
|
||||
$nextChunkIndex = (int) ($state['nextChunkIndex'] ?? 0);
|
||||
|
||||
if ($chunkCount <= 0 || $nextChunkIndex >= $chunkCount) {
|
||||
$this->repository->updateBatchSummary(
|
||||
(int) $state['batchId'],
|
||||
(int) $state['importedRows'],
|
||||
(int) $state['warningsCount'],
|
||||
'complete'
|
||||
);
|
||||
$this->repository->pruneExceptBatch((int) $state['batchId']);
|
||||
$state['status'] = 'completed';
|
||||
$state['done'] = true;
|
||||
$state['message'] = 'Import complete.';
|
||||
|
||||
return $state;
|
||||
}
|
||||
|
||||
$chunk = $this->reader->readChunkFromCache((string) $state['cacheBasePath'], $nextChunkIndex);
|
||||
$rows = is_array($chunk['rows'] ?? null) ? $chunk['rows'] : [];
|
||||
|
||||
$processedInChunk = 0;
|
||||
$this->repository->beginTransaction();
|
||||
try {
|
||||
foreach ($rows as $row) {
|
||||
if (!is_array($row) || !isset($row['row_number'], $row['data']) || !is_array($row['data'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$processedInChunk++;
|
||||
$rowNumber = (int) $row['row_number'];
|
||||
$sourceRow = $row['data'];
|
||||
|
||||
try {
|
||||
$processed = $this->processor->process($sourceRow);
|
||||
$this->repository->insertRow(
|
||||
(int) $state['batchId'],
|
||||
$rowNumber,
|
||||
$sourceRow,
|
||||
$processed['normalized'],
|
||||
$processed['calculated'],
|
||||
$processed['merged']
|
||||
);
|
||||
$state['importedRows']++;
|
||||
} catch (Throwable $throwable) {
|
||||
$state['warningsCount']++;
|
||||
if (count($state['warnings']) < 100) {
|
||||
$state['warnings'][] = sprintf('Row %d skipped: %s', $rowNumber, $throwable->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$state['processedRows'] = (int) ($state['processedRows'] ?? 0) + $processedInChunk;
|
||||
$state['nextChunkIndex'] = $nextChunkIndex + 1;
|
||||
|
||||
$isComplete = ((int) $state['nextChunkIndex']) >= $chunkCount;
|
||||
$this->repository->updateBatchSummary(
|
||||
(int) $state['batchId'],
|
||||
(int) $state['importedRows'],
|
||||
(int) $state['warningsCount'],
|
||||
$isComplete ? 'complete' : 'processing'
|
||||
);
|
||||
$this->repository->commit();
|
||||
|
||||
if ($isComplete) {
|
||||
$this->repository->pruneExceptBatch((int) $state['batchId']);
|
||||
$state['status'] = 'completed';
|
||||
$state['done'] = true;
|
||||
$state['message'] = 'Import complete.';
|
||||
} else {
|
||||
$state['status'] = 'running';
|
||||
$state['done'] = false;
|
||||
$state['message'] = sprintf('Processed %d row(s) in the latest chunk.', $processedInChunk);
|
||||
}
|
||||
} catch (Throwable $throwable) {
|
||||
$this->repository->rollBack();
|
||||
throw $throwable;
|
||||
}
|
||||
} catch (Throwable $throwable) {
|
||||
$state['status'] = 'error';
|
||||
$state['done'] = true;
|
||||
$state['message'] = 'Import failed: ' . $throwable->getMessage();
|
||||
$state['error'] = $throwable->getMessage();
|
||||
}
|
||||
|
||||
return $state;
|
||||
}
|
||||
|
||||
private function processLegacyJobState(array $state, int $chunkSize): array
|
||||
{
|
||||
if (($state['status'] ?? 'pending') === 'completed' || ($state['status'] ?? 'pending') === 'error') {
|
||||
return $state;
|
||||
}
|
||||
|
||||
try {
|
||||
if (!isset($state['batchId'])) {
|
||||
$headerInfo = $this->reader->detectHeader((string) $state['filePath'], self::REQUIRED_COLUMNS);
|
||||
$this->reader->validateRequiredColumns($headerInfo['headers'], self::REQUIRED_COLUMNS);
|
||||
|
||||
$state['batchId'] = $this->repository->createBatch(
|
||||
(string) $state['sourceFilename'],
|
||||
$headerInfo['headers'],
|
||||
$this->processor->calculatedHeaders(),
|
||||
'processing'
|
||||
);
|
||||
$state['headers'] = $headerInfo['headers'];
|
||||
$state['headerLookup'] = $headerInfo['header_lookup'];
|
||||
$state['headerRowIndex'] = (int) $headerInfo['header_row_index'];
|
||||
$state['highestDataRow'] = (int) $headerInfo['highest_data_row'];
|
||||
$state['nextRow'] = ((int) $headerInfo['header_row_index']) + 1;
|
||||
$state['totalRows'] = max(0, (int) $headerInfo['highest_data_row'] - (int) $headerInfo['header_row_index']);
|
||||
$state['importedRows'] = (int) ($state['importedRows'] ?? 0);
|
||||
$state['warningsCount'] = (int) ($state['warningsCount'] ?? 0);
|
||||
$state['warnings'] = is_array($state['warnings'] ?? null) ? array_values($state['warnings']) : [];
|
||||
}
|
||||
|
||||
$nextRow = (int) ($state['nextRow'] ?? 0);
|
||||
$highestDataRow = (int) ($state['highestDataRow'] ?? 0);
|
||||
|
||||
if ($nextRow > $highestDataRow) {
|
||||
$this->repository->updateBatchSummary(
|
||||
(int) $state['batchId'],
|
||||
(int) ($state['importedRows'] ?? 0),
|
||||
(int) ($state['warningsCount'] ?? 0),
|
||||
'complete'
|
||||
);
|
||||
$this->repository->pruneExceptBatch((int) $state['batchId']);
|
||||
$state['status'] = 'completed';
|
||||
$state['done'] = true;
|
||||
$state['message'] = 'Import complete.';
|
||||
|
||||
return $state;
|
||||
}
|
||||
|
||||
$rowsInWindow = max(0, min($chunkSize, $highestDataRow - $nextRow + 1));
|
||||
$chunk = $this->reader->readChunk(
|
||||
(string) $state['filePath'],
|
||||
$nextRow,
|
||||
$chunkSize,
|
||||
$state['headerLookup']
|
||||
);
|
||||
|
||||
$processedInChunk = 0;
|
||||
$this->repository->beginTransaction();
|
||||
try {
|
||||
foreach ($chunk['rows'] as $row) {
|
||||
$processedInChunk++;
|
||||
$rowNumber = (int) $row['row_number'];
|
||||
$sourceRow = $row['data'];
|
||||
|
||||
try {
|
||||
$processed = $this->processor->process($sourceRow);
|
||||
$this->repository->insertRow(
|
||||
(int) $state['batchId'],
|
||||
$rowNumber,
|
||||
$sourceRow,
|
||||
$processed['normalized'],
|
||||
$processed['calculated'],
|
||||
$processed['merged']
|
||||
);
|
||||
$state['importedRows'] = (int) ($state['importedRows'] ?? 0) + 1;
|
||||
} catch (Throwable $throwable) {
|
||||
$state['warningsCount'] = (int) ($state['warningsCount'] ?? 0) + 1;
|
||||
if (!isset($state['warnings']) || !is_array($state['warnings'])) {
|
||||
$state['warnings'] = [];
|
||||
}
|
||||
if (count($state['warnings']) < 100) {
|
||||
$state['warnings'][] = sprintf('Row %d skipped: %s', $rowNumber, $throwable->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$state['nextRow'] = $nextRow + $rowsInWindow;
|
||||
$state['processedRows'] = (int) ($state['processedRows'] ?? 0) + $rowsInWindow;
|
||||
|
||||
$isComplete = (int) $state['nextRow'] > $highestDataRow;
|
||||
$this->repository->updateBatchSummary(
|
||||
(int) $state['batchId'],
|
||||
(int) $state['importedRows'],
|
||||
(int) $state['warningsCount'],
|
||||
$isComplete ? 'complete' : 'processing'
|
||||
);
|
||||
$this->repository->commit();
|
||||
|
||||
if ($isComplete) {
|
||||
$this->repository->pruneExceptBatch((int) $state['batchId']);
|
||||
$state['status'] = 'completed';
|
||||
$state['done'] = true;
|
||||
$state['message'] = 'Import complete.';
|
||||
} else {
|
||||
$state['status'] = 'running';
|
||||
$state['done'] = false;
|
||||
$state['message'] = sprintf('Processed %d row(s) in the latest chunk.', $processedInChunk);
|
||||
}
|
||||
} catch (Throwable $throwable) {
|
||||
$this->repository->rollBack();
|
||||
throw $throwable;
|
||||
}
|
||||
} catch (Throwable $throwable) {
|
||||
$state['status'] = 'error';
|
||||
$state['done'] = true;
|
||||
$state['message'] = 'Import failed: ' . $throwable->getMessage();
|
||||
$state['error'] = $throwable->getMessage();
|
||||
}
|
||||
|
||||
return $state;
|
||||
}
|
||||
|
||||
public function latestBatch(): ?array
|
||||
{
|
||||
return $this->repository->getLatestBatch();
|
||||
}
|
||||
|
||||
public function clear(): void
|
||||
{
|
||||
$this->repository->beginTransaction();
|
||||
try {
|
||||
$this->repository->clearAll();
|
||||
$this->repository->commit();
|
||||
} catch (Throwable $throwable) {
|
||||
$this->repository->rollBack();
|
||||
throw new RuntimeException('Clear failed: ' . $throwable->getMessage(), 0, $throwable);
|
||||
}
|
||||
}
|
||||
|
||||
public function getPreview(int $batchId, int $limit, int $offset, string $sortDirection = 'asc'): array
|
||||
{
|
||||
return $this->repository->paginateRows($batchId, $limit, $offset, $sortDirection);
|
||||
}
|
||||
|
||||
public function getFilteredPreview(
|
||||
int $batchId,
|
||||
int $limit,
|
||||
int $offset,
|
||||
string $query,
|
||||
array $tableHeaders = [],
|
||||
array $columnSearches = [],
|
||||
?int $orderColumn = null,
|
||||
string $sortDirection = 'asc'
|
||||
): array
|
||||
{
|
||||
return $this->repository->paginateRowsFiltered(
|
||||
$batchId,
|
||||
$limit,
|
||||
$offset,
|
||||
$query,
|
||||
$tableHeaders,
|
||||
$columnSearches,
|
||||
$orderColumn,
|
||||
$sortDirection
|
||||
);
|
||||
}
|
||||
|
||||
public function getGridRows(
|
||||
int $batchId,
|
||||
int $limit,
|
||||
int $offset,
|
||||
string $query = '',
|
||||
array $tableHeaders = [],
|
||||
array $filterModel = [],
|
||||
array $sortModel = []
|
||||
): array
|
||||
{
|
||||
return $this->repository->paginateRowsGrid(
|
||||
$batchId,
|
||||
$limit,
|
||||
$offset,
|
||||
$query,
|
||||
$tableHeaders,
|
||||
$filterModel,
|
||||
$sortModel
|
||||
);
|
||||
}
|
||||
|
||||
public function countRowsForBatch(int $batchId): int
|
||||
{
|
||||
return $this->repository->countRowsForBatch($batchId);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array<string, mixed>>
|
||||
*/
|
||||
public function getExportRows(int $batchId, int $limit, int $offset): array
|
||||
{
|
||||
return $this->repository->exportRowsForBatch($batchId, $limit, $offset);
|
||||
}
|
||||
|
||||
public function getDistinctValuesForBatch(int $batchId, string $header, int $limit = 200): array
|
||||
{
|
||||
return $this->repository->getDistinctValuesForBatch($batchId, $header, $limit);
|
||||
}
|
||||
|
||||
public function getDistinctValuesForBatchGrid(
|
||||
int $batchId,
|
||||
array $tableHeaders,
|
||||
string $columnKey,
|
||||
string $distinctSearch = '',
|
||||
int $limit = 500
|
||||
): array
|
||||
{
|
||||
return $this->repository->getDistinctValuesForBatchGrid($batchId, $tableHeaders, $columnKey, $distinctSearch, $limit);
|
||||
}
|
||||
|
||||
public function countFilteredRowsForBatch(int $batchId, string $query, array $tableHeaders = [], array $columnSearches = []): int
|
||||
{
|
||||
return $this->repository->countRowsForBatchFiltered($batchId, $query, $tableHeaders, $columnSearches);
|
||||
}
|
||||
|
||||
public function countGridRows(
|
||||
int $batchId,
|
||||
string $query = '',
|
||||
array $tableHeaders = [],
|
||||
array $filterModel = []
|
||||
): int
|
||||
{
|
||||
return $this->repository->countRowsForBatchGrid($batchId, $query, $tableHeaders, $filterModel);
|
||||
}
|
||||
|
||||
public function chunkSize(): int
|
||||
{
|
||||
return $this->chunkSize;
|
||||
}
|
||||
|
||||
public function deleteCache(string $cacheBasePath): void
|
||||
{
|
||||
$this->reader->deleteCache($cacheBasePath);
|
||||
}
|
||||
|
||||
private function cacheBasePath(array $state): string
|
||||
{
|
||||
$token = preg_replace('/[^a-zA-Z0-9_-]/', '', (string) ($state['token'] ?? ''));
|
||||
if ($token === '') {
|
||||
throw new RuntimeException('Missing import token.');
|
||||
}
|
||||
|
||||
return sys_get_temp_dir() . '/warner-import-cache/' . $token;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,456 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
use OpenSpout\Reader\CSV\Reader as CsvReader;
|
||||
use OpenSpout\Reader\ODS\Reader as OdsReader;
|
||||
use OpenSpout\Reader\ReaderInterface as SpoutReaderInterface;
|
||||
use OpenSpout\Reader\XLSX\Reader as XlsxReader;
|
||||
use PhpOffice\PhpSpreadsheet\IOFactory;
|
||||
use RuntimeException;
|
||||
use Throwable;
|
||||
|
||||
final class SpreadsheetReader
|
||||
{
|
||||
public function detectHeader(string $filePath, array $expectedHeaders = [], int $scanChunkSize = 100, int $maxScanRows = 5000): array
|
||||
{
|
||||
for ($startRow = 1; $startRow <= $maxScanRows; $startRow += $scanChunkSize) {
|
||||
$endRow = min($maxScanRows, $startRow + $scanChunkSize - 1);
|
||||
$window = $this->loadWindow($filePath, $startRow, $endRow);
|
||||
|
||||
foreach ($window['rows'] as $rowIndex => $row) {
|
||||
$normalizedRow = [];
|
||||
foreach ($row as $columnLetter => $cellValue) {
|
||||
$normalizedRow[$columnLetter] = $this->normalizeHeader($cellValue);
|
||||
}
|
||||
|
||||
if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$headers = [];
|
||||
$headerLookup = [];
|
||||
foreach ($normalizedRow as $columnLetter => $headerValue) {
|
||||
if ($headerValue === '' || $headerValue === 'Revenue Type') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$headers[] = $headerValue;
|
||||
$headerLookup[$columnLetter] = $headerValue;
|
||||
}
|
||||
|
||||
return [
|
||||
'header_row_index' => (int) $rowIndex,
|
||||
'headers' => $headers,
|
||||
'header_lookup' => $headerLookup,
|
||||
'highest_data_row' => (int) $window['highest_data_row'],
|
||||
];
|
||||
}
|
||||
|
||||
if ((int) $window['highest_data_row'] < $endRow) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
throw new RuntimeException('Could not locate the header row in the uploaded workbook.');
|
||||
}
|
||||
|
||||
public function readChunk(string $filePath, int $startRow, int $chunkSize, array $headerLookup): array
|
||||
{
|
||||
$endRow = $startRow + $chunkSize - 1;
|
||||
$window = $this->loadWindow($filePath, $startRow, $endRow);
|
||||
$rows = [];
|
||||
|
||||
foreach ($window['rows'] as $rowIndex => $row) {
|
||||
$mappedRow = [];
|
||||
foreach ($headerLookup as $columnLetter => $header) {
|
||||
$mappedRow[$header] = $this->normalizeCellValue($row[$columnLetter] ?? null);
|
||||
}
|
||||
|
||||
if ($this->isBlankRow($mappedRow)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$rows[] = [
|
||||
'row_number' => (int) $rowIndex,
|
||||
'data' => $mappedRow,
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'rows' => $rows,
|
||||
'highest_data_row' => (int) $window['highest_data_row'],
|
||||
];
|
||||
}
|
||||
|
||||
public function read(string $filePath, array $expectedHeaders = []): array
|
||||
{
|
||||
$header = $this->detectHeader($filePath, $expectedHeaders);
|
||||
$allRows = [];
|
||||
$nextRow = $header['header_row_index'] + 1;
|
||||
|
||||
while ($nextRow <= $header['highest_data_row']) {
|
||||
$chunk = $this->readChunk($filePath, $nextRow, 500, $header['header_lookup']);
|
||||
$allRows = array_merge($allRows, $chunk['rows']);
|
||||
$nextRow += 500;
|
||||
}
|
||||
|
||||
return [
|
||||
'headers' => $header['headers'],
|
||||
'rows' => $allRows,
|
||||
];
|
||||
}
|
||||
|
||||
public function buildChunkCache(string $filePath, string $cacheBasePath, array $expectedHeaders = [], int $chunkSize = 50): array
|
||||
{
|
||||
if ($chunkSize < 1) {
|
||||
throw new RuntimeException('Chunk size must be at least 1.');
|
||||
}
|
||||
|
||||
$reader = $this->createSpoutReader($filePath);
|
||||
$metaPath = $this->cacheMetaPath($cacheBasePath);
|
||||
$rowsPath = $this->cacheRowsPath($cacheBasePath);
|
||||
$cacheDir = dirname($metaPath);
|
||||
if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) {
|
||||
throw new RuntimeException('Unable to create the spreadsheet cache directory.');
|
||||
}
|
||||
|
||||
$rowsHandle = fopen($rowsPath, 'wb');
|
||||
if ($rowsHandle === false) {
|
||||
throw new RuntimeException('Unable to create the spreadsheet cache file.');
|
||||
}
|
||||
|
||||
$headerFound = false;
|
||||
$rowIndex = 0;
|
||||
$chunkRows = [];
|
||||
$chunkOffsets = [];
|
||||
$totalRows = 0;
|
||||
$headerRowIndex = 0;
|
||||
$headers = [];
|
||||
$headerLookup = [];
|
||||
|
||||
try {
|
||||
$reader->open($filePath);
|
||||
|
||||
foreach ($reader->getSheetIterator() as $sheet) {
|
||||
foreach ($sheet->getRowIterator() as $row) {
|
||||
$rowIndex++;
|
||||
$values = $row->toArray();
|
||||
|
||||
if (!$headerFound) {
|
||||
$normalizedRow = [];
|
||||
foreach ($values as $columnIndex => $cellValue) {
|
||||
$normalizedRow[$columnIndex] = $this->normalizeHeader($cellValue);
|
||||
}
|
||||
|
||||
if (!$this->isHeaderRow($normalizedRow, $expectedHeaders)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($normalizedRow as $columnIndex => $headerValue) {
|
||||
if ($headerValue === '' || $headerValue === 'Revenue Type') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$headers[] = $headerValue;
|
||||
$headerLookup[(int) $columnIndex] = $headerValue;
|
||||
}
|
||||
|
||||
$headerRowIndex = $rowIndex;
|
||||
$headerFound = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
$mappedRow = [];
|
||||
foreach ($headerLookup as $columnIndex => $header) {
|
||||
$mappedRow[$header] = $this->normalizeCellValue($values[$columnIndex] ?? null);
|
||||
}
|
||||
|
||||
if ($this->isBlankRow($mappedRow)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$chunkRows[] = [
|
||||
'row_number' => $rowIndex,
|
||||
'data' => $mappedRow,
|
||||
];
|
||||
$totalRows++;
|
||||
|
||||
if (count($chunkRows) >= $chunkSize) {
|
||||
$chunkOffsets[] = ftell($rowsHandle);
|
||||
$this->writeChunkRow($rowsHandle, $chunkRows);
|
||||
$chunkRows = [];
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
} catch (Throwable $throwable) {
|
||||
throw new RuntimeException('Unable to build spreadsheet cache: ' . $throwable->getMessage(), 0, $throwable);
|
||||
} finally {
|
||||
$reader->close();
|
||||
fclose($rowsHandle);
|
||||
}
|
||||
|
||||
if (!$headerFound) {
|
||||
@unlink($rowsPath);
|
||||
throw new RuntimeException('Could not locate the header row in the uploaded workbook.');
|
||||
}
|
||||
|
||||
if ($chunkRows !== []) {
|
||||
$chunkOffsets[] = file_exists($rowsPath) ? filesize($rowsPath) : 0;
|
||||
$rowsHandle = fopen($rowsPath, 'ab');
|
||||
if ($rowsHandle === false) {
|
||||
throw new RuntimeException('Unable to reopen the spreadsheet cache file.');
|
||||
}
|
||||
|
||||
try {
|
||||
$this->writeChunkRow($rowsHandle, $chunkRows);
|
||||
} finally {
|
||||
fclose($rowsHandle);
|
||||
}
|
||||
}
|
||||
|
||||
$cache = [
|
||||
'file_path' => $filePath,
|
||||
'headers' => $headers,
|
||||
'header_lookup' => $headerLookup,
|
||||
'header_row_index' => $headerRowIndex,
|
||||
'total_rows' => $totalRows,
|
||||
'chunk_size' => $chunkSize,
|
||||
'chunk_count' => count($chunkOffsets),
|
||||
'chunk_offsets' => $chunkOffsets,
|
||||
];
|
||||
|
||||
if (file_put_contents(
|
||||
$metaPath,
|
||||
json_encode($cache, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)
|
||||
) === false) {
|
||||
throw new RuntimeException('Unable to write the spreadsheet cache metadata.');
|
||||
}
|
||||
|
||||
return $cache;
|
||||
}
|
||||
|
||||
public function readChunkFromCache(string $cacheBasePath, int $chunkIndex): array
|
||||
{
|
||||
$cache = $this->readCacheMeta($cacheBasePath);
|
||||
$chunkOffsets = $cache['chunk_offsets'] ?? [];
|
||||
if (!is_array($chunkOffsets) || !array_key_exists($chunkIndex, $chunkOffsets)) {
|
||||
return [
|
||||
'rows' => [],
|
||||
'cache' => $cache,
|
||||
];
|
||||
}
|
||||
|
||||
$rowsPath = $this->cacheRowsPath($cacheBasePath);
|
||||
$rowsHandle = fopen($rowsPath, 'rb');
|
||||
if ($rowsHandle === false) {
|
||||
throw new RuntimeException('Unable to open the spreadsheet cache file.');
|
||||
}
|
||||
|
||||
try {
|
||||
if (fseek($rowsHandle, (int) $chunkOffsets[$chunkIndex]) !== 0) {
|
||||
throw new RuntimeException('Unable to seek to the requested spreadsheet chunk.');
|
||||
}
|
||||
|
||||
$line = fgets($rowsHandle);
|
||||
if ($line === false) {
|
||||
throw new RuntimeException('Unable to read the requested spreadsheet chunk.');
|
||||
}
|
||||
|
||||
$rows = json_decode(trim($line), true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($rows)) {
|
||||
throw new RuntimeException('Spreadsheet chunk data is invalid.');
|
||||
}
|
||||
} finally {
|
||||
fclose($rowsHandle);
|
||||
}
|
||||
|
||||
return [
|
||||
'rows' => $rows,
|
||||
'cache' => $cache,
|
||||
];
|
||||
}
|
||||
|
||||
public function deleteCache(string $cacheBasePath): void
|
||||
{
|
||||
@unlink($this->cacheMetaPath($cacheBasePath));
|
||||
@unlink($this->cacheRowsPath($cacheBasePath));
|
||||
}
|
||||
|
||||
public function validateRequiredColumns(array $headers, array $requiredColumns): void
|
||||
{
|
||||
$missing = [];
|
||||
$headerMap = array_fill_keys($headers, true);
|
||||
|
||||
foreach ($requiredColumns as $requiredColumn) {
|
||||
if (!isset($headerMap[$requiredColumn])) {
|
||||
$missing[] = $requiredColumn;
|
||||
}
|
||||
}
|
||||
|
||||
if ($missing !== []) {
|
||||
throw new RuntimeException('Missing required column(s): ' . implode(', ', $missing));
|
||||
}
|
||||
}
|
||||
|
||||
private function normalizeHeader(mixed $value): string
|
||||
{
|
||||
if ($value === null) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return trim((string) $value);
|
||||
}
|
||||
|
||||
private function isHeaderRow(array $row, array $expectedHeaders): bool
|
||||
{
|
||||
$nonEmptyValues = array_values(array_filter($row, static fn (?string $value): bool => $value !== null && $value !== ''));
|
||||
if ($nonEmptyValues === []) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($expectedHeaders === []) {
|
||||
return count($nonEmptyValues) >= 2;
|
||||
}
|
||||
|
||||
$lookup = array_fill_keys($nonEmptyValues, true);
|
||||
foreach ($expectedHeaders as $expectedHeader) {
|
||||
if (!isset($lookup[$expectedHeader])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function loadWindow(string $filePath, int $startRow, int $endRow): array
|
||||
{
|
||||
try {
|
||||
$reader = IOFactory::createReaderForFile($filePath);
|
||||
$reader->setReadDataOnly(true);
|
||||
|
||||
$sheetNames = $reader->listWorksheetNames($filePath);
|
||||
if ($sheetNames !== []) {
|
||||
$reader->setLoadSheetsOnly([$sheetNames[0]]);
|
||||
}
|
||||
|
||||
$reader->setReadFilter(new ChunkReadFilter($startRow, $endRow));
|
||||
$spreadsheet = $reader->load($filePath);
|
||||
} catch (Throwable $throwable) {
|
||||
throw new RuntimeException('Unable to read the uploaded Excel file: ' . $throwable->getMessage(), 0, $throwable);
|
||||
}
|
||||
|
||||
try {
|
||||
$worksheet = $spreadsheet->getSheet(0);
|
||||
$rows = $worksheet->toArray(null, false, true, true);
|
||||
|
||||
return [
|
||||
'rows' => $rows,
|
||||
'highest_data_row' => $worksheet->getHighestDataRow(),
|
||||
];
|
||||
} finally {
|
||||
$spreadsheet->disconnectWorksheets();
|
||||
unset($spreadsheet);
|
||||
}
|
||||
}
|
||||
|
||||
private function createSpoutReader(string $filePath): SpoutReaderInterface
|
||||
{
|
||||
$extension = strtolower(pathinfo($filePath, PATHINFO_EXTENSION));
|
||||
|
||||
return match ($extension) {
|
||||
'csv' => new CsvReader(),
|
||||
'ods' => new OdsReader(),
|
||||
'xlsx', 'xlsm' => new XlsxReader(),
|
||||
'xls' => throw new RuntimeException('XLS files are not supported by the streaming reader. Please save the workbook as XLSX before importing.'),
|
||||
default => new XlsxReader(),
|
||||
};
|
||||
}
|
||||
|
||||
private function cacheMetaPath(string $cacheBasePath): string
|
||||
{
|
||||
return rtrim($cacheBasePath, '/\\') . '.meta.json';
|
||||
}
|
||||
|
||||
private function cacheRowsPath(string $cacheBasePath): string
|
||||
{
|
||||
return rtrim($cacheBasePath, '/\\') . '.rows.ndjson';
|
||||
}
|
||||
|
||||
private function readCacheMeta(string $cacheBasePath): array
|
||||
{
|
||||
$metaPath = $this->cacheMetaPath($cacheBasePath);
|
||||
if (!is_file($metaPath)) {
|
||||
throw new RuntimeException('Spreadsheet cache metadata is missing. Please restart the import.');
|
||||
}
|
||||
|
||||
$contents = file_get_contents($metaPath);
|
||||
if ($contents === false || $contents === '') {
|
||||
throw new RuntimeException('Spreadsheet cache metadata is unreadable. Please restart the import.');
|
||||
}
|
||||
|
||||
$cache = json_decode($contents, true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($cache)) {
|
||||
throw new RuntimeException('Spreadsheet cache metadata is invalid. Please restart the import.');
|
||||
}
|
||||
|
||||
return $cache;
|
||||
}
|
||||
|
||||
private function writeChunkRow($handle, array $chunkRows): void
|
||||
{
|
||||
$json = json_encode($chunkRows, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
||||
if (fwrite($handle, $json . PHP_EOL) === false) {
|
||||
throw new RuntimeException('Unable to write spreadsheet cache chunk data.');
|
||||
}
|
||||
}
|
||||
|
||||
public function normalizeCellValue(mixed $value): ?string
|
||||
{
|
||||
if ($value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($value instanceof \DateTimeInterface) {
|
||||
return $value->format('Y-m-d H:i:s');
|
||||
}
|
||||
|
||||
if (is_bool($value)) {
|
||||
return $value ? '1' : '0';
|
||||
}
|
||||
|
||||
if (is_int($value)) {
|
||||
return (string) $value;
|
||||
}
|
||||
|
||||
if (is_float($value)) {
|
||||
if (fmod($value, 1.0) === 0.0) {
|
||||
return (string) (int) $value;
|
||||
}
|
||||
|
||||
return rtrim(rtrim(sprintf('%.15F', $value), '0'), '.');
|
||||
}
|
||||
|
||||
$string = trim((string) $value);
|
||||
if ($string === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $string;
|
||||
}
|
||||
|
||||
private function isBlankRow(array $row): bool
|
||||
{
|
||||
foreach ($row as $value) {
|
||||
if ($value !== null && $value !== '') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Import;
|
||||
|
||||
final class SpreadsheetRowProcessor
|
||||
{
|
||||
private const CALCULATED_HEADERS = [
|
||||
'Actual/Accrual',
|
||||
'US/ex-US sale',
|
||||
'Contracting Party Filtered',
|
||||
'String',
|
||||
'2110 Applicable X201',
|
||||
'Class',
|
||||
];
|
||||
|
||||
private const X201_ELIGIBLE_STRINGS = [
|
||||
'1756YYFDI',
|
||||
'1756YYFLI',
|
||||
'1756YYFPA',
|
||||
'1756YYFWO',
|
||||
'1756YYDDI',
|
||||
'1756YYDWO',
|
||||
'1756YYDLI',
|
||||
'1756YYDPA',
|
||||
'1100NYDDI',
|
||||
'1100NYDWO',
|
||||
'1100YYDWO',
|
||||
'1100YYFDI',
|
||||
'1100YYFLI',
|
||||
'1100YYFPA',
|
||||
'1100YYFWO',
|
||||
'1100NYDLI',
|
||||
'1100NYDPA',
|
||||
'1100YYDLI',
|
||||
'1100YYDDI',
|
||||
'1100YYDPA',
|
||||
'1902NYDDI',
|
||||
'1902NYDLI',
|
||||
'1902NYDWO',
|
||||
'1902YYFDI',
|
||||
'1902YYFLI',
|
||||
'1902YYFPA',
|
||||
'1902YYFWO',
|
||||
'1902YYDDI',
|
||||
'1902YYDWO',
|
||||
'1902NYDPA',
|
||||
'1902YYDLI',
|
||||
'1902YYDPA',
|
||||
'LOCALYYDWO',
|
||||
'LOCALYYDDI',
|
||||
'LOCALYYDLI',
|
||||
];
|
||||
|
||||
public function calculatedHeaders(): array
|
||||
{
|
||||
return self::CALCULATED_HEADERS;
|
||||
}
|
||||
|
||||
public function process(array $row): array
|
||||
{
|
||||
$normalized = $this->normalize($row);
|
||||
$calculated = $this->calculateDerivedFields($normalized);
|
||||
|
||||
return [
|
||||
'normalized' => $normalized,
|
||||
'calculated' => $calculated,
|
||||
'merged' => array_merge($normalized, $calculated),
|
||||
];
|
||||
}
|
||||
|
||||
private function normalize(array $row): array
|
||||
{
|
||||
$normalized = [];
|
||||
foreach ($row as $header => $value) {
|
||||
$normalized[$header] = $this->normalizeValue($value);
|
||||
}
|
||||
|
||||
return $normalized;
|
||||
}
|
||||
|
||||
private function normalizeValue(mixed $value): ?string
|
||||
{
|
||||
if ($value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$value = trim((string) $value);
|
||||
return $value === '' ? null : $value;
|
||||
}
|
||||
|
||||
private function calculateDerivedFields(array $row): array
|
||||
{
|
||||
$contractingParty = $this->normalizeCode($row['Contracting Party'] ?? null);
|
||||
$localSellingCompany = $this->normalizeCode($row['Local Selling Company (S/T)'] ?? null);
|
||||
$rightsHolderLiableFlag = $this->normalizeFlag($row['Rights Holder Liable Flag'] ?? null);
|
||||
$globalTransferPricingFlag = $this->normalizeFlag($row['Global Transfer Pricing Flag'] ?? null);
|
||||
$domesticForeignFlag = $this->normalizeFlag($row['Domestic/ Foreign Flag'] ?? null);
|
||||
$ownershipType = $this->normalizeValue($row['Ownership Type'] ?? null) ?? '';
|
||||
|
||||
// Business rule: Actual vs accrual is driven by the raw Accrual Category value.
|
||||
$actualAccrual = (($row['Accrual Category'] ?? null) === '##') ? 'Actual' : 'Accrual';
|
||||
|
||||
// Business rule: US/ex-US depends on the local selling company range.
|
||||
$usExUs = $this->inRange($localSellingCompany, 1700, 1809) ? 'US' : 'ex-US';
|
||||
|
||||
// Business rule: Contracting Party is narrowed to the special codes; all others become LOCAL.
|
||||
$contractingPartyFiltered = in_array($contractingParty, ['1756', '1100', '1902'], true)
|
||||
? $contractingParty
|
||||
: 'LOCAL';
|
||||
|
||||
// Business rule: String is a direct concatenation of the filtered contracting party plus the flags.
|
||||
$string = $contractingPartyFiltered
|
||||
. $rightsHolderLiableFlag
|
||||
. $globalTransferPricingFlag
|
||||
. $domesticForeignFlag
|
||||
. $ownershipType;
|
||||
|
||||
// Business rule: 2110 applicability is a fixed whitelist lookup against the derived String.
|
||||
$applicableX201 = in_array($string, self::X201_ELIGIBLE_STRINGS, true) ? 'X' : '';
|
||||
|
||||
// Business rule: Class uses ordered precedence. The first matching rule wins.
|
||||
$class = $this->calculateClass($contractingParty, $localSellingCompany, $rightsHolderLiableFlag, $globalTransferPricingFlag, $applicableX201, $contractingPartyFiltered);
|
||||
|
||||
return [
|
||||
'Actual/Accrual' => $actualAccrual,
|
||||
'US/ex-US sale' => $usExUs,
|
||||
'Contracting Party Filtered' => $contractingPartyFiltered,
|
||||
'String' => $string,
|
||||
'2110 Applicable X201' => $applicableX201,
|
||||
'Class' => $class,
|
||||
];
|
||||
}
|
||||
|
||||
private function calculateClass(
|
||||
?string $contractingParty,
|
||||
?string $localSellingCompany,
|
||||
string $rightsHolderLiableFlag,
|
||||
string $globalTransferPricingFlag,
|
||||
string $applicableX201,
|
||||
string $contractingPartyFiltered
|
||||
): string {
|
||||
if ($contractingParty === '1902') {
|
||||
return 'A';
|
||||
}
|
||||
|
||||
if (
|
||||
$contractingParty === '1756'
|
||||
&& $rightsHolderLiableFlag === 'Y'
|
||||
&& !$this->inRange($localSellingCompany, 1700, 1808)
|
||||
) {
|
||||
return 'B';
|
||||
}
|
||||
|
||||
if ($contractingParty === '1756' && $this->inRange($localSellingCompany, 1700, 1808) && $applicableX201 === 'X') {
|
||||
return 'C';
|
||||
}
|
||||
|
||||
if ($contractingParty === '1756' && $this->inRange($localSellingCompany, 1700, 1808) && $applicableX201 !== 'X') {
|
||||
return 'D';
|
||||
}
|
||||
|
||||
if ($contractingParty === '1100' && !$this->inRange($localSellingCompany, 1700, 1808)) {
|
||||
return 'E';
|
||||
}
|
||||
|
||||
if ($contractingParty === '1100' && $this->inRange($localSellingCompany, 1700, 1808)) {
|
||||
return 'F';
|
||||
}
|
||||
|
||||
if ($contractingPartyFiltered === 'LOCAL') {
|
||||
return 'G';
|
||||
}
|
||||
|
||||
if (
|
||||
$contractingParty === '1756'
|
||||
&& $rightsHolderLiableFlag === 'N'
|
||||
&& $globalTransferPricingFlag === 'Y'
|
||||
&& !$this->inRange($localSellingCompany, 1700, 1808)
|
||||
) {
|
||||
return 'H';
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
private function normalizeCode(mixed $value): ?string
|
||||
{
|
||||
$value = $this->normalizeValue($value);
|
||||
if ($value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (is_numeric($value) && (float) $value == (int) $value) {
|
||||
return (string) (int) $value;
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
|
||||
private function normalizeFlag(mixed $value): string
|
||||
{
|
||||
$normalized = $this->normalizeValue($value);
|
||||
return $normalized ?? '';
|
||||
}
|
||||
|
||||
private function inRange(?string $value, int $min, int $max): bool
|
||||
{
|
||||
if ($value === null || $value === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!is_numeric($value)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$number = (int) $value;
|
||||
return $number >= $min && $number <= $max;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user