From 1a249c3caabf1dac338e33a6eda5894bceb07329 Mon Sep 17 00:00:00 2001 From: Brad Lance Date: Sun, 5 Jul 2026 20:19:34 -0500 Subject: [PATCH] Auto-add missing entity.json keys as null; null now means ignore, not error On each run the engine compares entity.json against the full set of known keys, adds any that are absent with a null value, and writes the file back - so nobody has to hand-type key names (the typo class of config error goes away). A null/blank value now means "this entity doesn't use this": files that would need it are ignored with a yellow notice and left in place, instead of counting as failures. Insurance-unconfigured no longer skips the patient step for that practice (was continue, now else-branch). --- README.md | 29 +++++++++-------- engine.ps1 | 96 ++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 77 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index bbd35dc..04c913b 100644 --- a/README.md +++ b/README.md @@ -129,11 +129,13 @@ leave its login and every other path alone. folder name — so one `entity.json` can serve every practice as long as they all follow the same folder-naming pattern on the server. -**Every "Yes" field must be present in `entity.json`** unless *every single -practice* under the entity supplies it in its own `practice.json`. If a -required field is missing or blank, the run stops that practice with a -`CONFIG ERROR` message naming exactly which fields it couldn't find — nothing -uploads and nothing archives for that practice until it's fixed. +**Missing keys fix themselves:** on every run, the tool checks `entity.json` +for any keys it knows about that aren't in the file, adds them with a value +of `null`, and saves the file — so you never have to type key names by hand. +A `null` value means **"we don't use this — ignore it"**: any files that +would need that setting are left in place untouched (with a yellow notice in +the output), not treated as failures. To turn a feature on, just open +`entity.json` and replace the `null` with a real value in quotes. **Editing JSON — the rules that bite:** - Key names must match **exactly** (all lowercase, underscores): `host`, not @@ -207,16 +209,17 @@ those are entity-wide only. - **Files failed to upload** — they stay in the date folder. Fix the issue and run again. - **A date folder is still there after running** — one or more files failed. Check the output window. - **WinSCP error on startup** — run `setup.bat` again to re-download WinSCP. -- **`CONFIG ERROR: entity.json is missing or has blank: ...`** — the fields it - lists couldn't be found in that entity's `entity.json` (or the practice's - `practice.json`). Open the file and check those exact key names against the - example above — a typo in the key name counts as missing. +- **`... upload not configured (ftps.xyz is null/blank) - N file(s) ignored`** — + there are files to send, but the listed settings are still `null` (or blank) + in that entity's `entity.json`. If those files *should* upload, open + `entity.json` and fill in real values for the listed keys. If that upload + type genuinely doesn't apply to this entity, the message is normal and the + files just stay where they are. +- **`Added missing key(s) to entity.json ...`** — informational, not an error. + The tool added keys that were absent (as `null`) so they're ready to fill in. - **`No .pdf files in \` but the folder isn't empty** — the files in the - date folder are probably TIFs. Add `"insurance_file_type": "tif",` at the top + date folder are probably TIFs. Set `"insurance_file_type": "tif",` at the top level of that entity's `entity.json` (right under `workflow`). -- **Every field shows missing at once** — the section name itself is probably - wrong (must be exactly `ftps` / `sftp`), or the file's structure got damaged - while editing. Compare the overall shape against `Entities\EXAMPLE\entity.json`. - **The startup banner shows the wrong file type** — the banner line (`Entity: ... | File type: .pdf | ...`) shows what was actually read from `entity.json`, so it's the quickest way to confirm your edit took effect. diff --git a/engine.ps1 b/engine.ps1 index 8285a12..853d4c1 100644 --- a/engine.ps1 +++ b/engine.ps1 @@ -76,6 +76,39 @@ if (-not (Test-Path $entityConfigFile)) { } $entityConfig = Get-Content $entityConfigFile -Raw | ConvertFrom-Json + +# Add any missing keys to entity.json as null so they never have to be typed +# by hand (a null value means "not used - ignore"). Writes the file back only +# when something was actually added. +$configTemplate = [ordered]@{ + workflow = $null + insurance_file_type = $null + ftps = [ordered]@{ host = $null; port = $null; tls = $null; username = $null; password = $null; tiff_path = $null; pdf_path = $null } + sftp = [ordered]@{ host = $null; port = $null; username = $null; password = $null; patient_path = $null } +} +$addedKeys = @() +foreach ($topKey in $configTemplate.Keys) { + if ($null -eq $entityConfig.PSObject.Properties[$topKey]) { + $value = if ($configTemplate[$topKey] -is [System.Collections.Specialized.OrderedDictionary]) { + [PSCustomObject]$configTemplate[$topKey] + } else { $configTemplate[$topKey] } + $entityConfig | Add-Member -NotePropertyName $topKey -NotePropertyValue $value + $addedKeys += $topKey + } elseif ($configTemplate[$topKey] -is [System.Collections.Specialized.OrderedDictionary]) { + foreach ($subKey in $configTemplate[$topKey].Keys) { + if ($null -eq $entityConfig.$topKey.PSObject.Properties[$subKey]) { + $entityConfig.$topKey | Add-Member -NotePropertyName $subKey -NotePropertyValue $null + $addedKeys += "$topKey.$subKey" + } + } + } +} +if ($addedKeys.Count -gt 0) { + $entityConfig | ConvertTo-Json -Depth 10 | Set-Content -Path $entityConfigFile -Encoding UTF8 + Write-Host "Added missing key(s) to entity.json (as null = ignored): $($addedKeys -join ', ')" -ForegroundColor Yellow + Write-Info "CONFIG added missing keys to entity.json: $($addedKeys -join ',')" +} + $workflow = $entityConfig.workflow $insuranceExt = if ($entityConfig.insurance_file_type) { $entityConfig.insurance_file_type.TrimStart(".") } else { "pdf" } $export1 = Join-Path $EntityDir "Export1" @@ -269,39 +302,35 @@ foreach ($practice in $practices) { $missing = Get-MissingFields $config.ftps "ftps" @("host", "username", "password", "tiff_path", "pdf_path") if (($insFiles.Count -gt 0 -or $ptStmts.Count -gt 0) -and $missing.Count -gt 0) { - Write-Host " CONFIG ERROR: entity.json is missing or has blank: $($missing -join ', ')" -ForegroundColor Red - Write-Host " Check spelling of those keys in entity.json (and practice.json if present)." -ForegroundColor Yellow - Write-Log "error" "CONFIG entity=$entityName practice=$pracName missing=$($missing -join ',')" - $totalFail += $insFiles.Count + $ptStmts.Count - Write-Host " $today\ left for retry." -ForegroundColor Yellow - continue - } + Write-Host " Insurance upload not configured ($($missing -join ', ') is null/blank) - $($insFiles.Count + $ptStmts.Count) file(s) ignored, left in place." -ForegroundColor Yellow + Write-Info "IGNORE entity=$entityName practice=$pracName reason=unconfigured fields=$($missing -join ',')" + } else { + if ($insFiles.Count -gt 0) { + Write-Host " Insurance ($($insFiles.Count)) -> $insPath" -ForegroundColor White + $r = Invoke-FTPSUpload $config.ftps $insFiles $insPath $entityName $pracName + $insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail + } + if ($ptStmts.Count -gt 0) { + Write-Host " PT STMT ($($ptStmts.Count)) -> $pdfPath" -ForegroundColor White + $r = Invoke-FTPSUpload $config.ftps $ptStmts $pdfPath $entityName $pracName + $insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail + } - if ($insFiles.Count -gt 0) { - Write-Host " Insurance ($($insFiles.Count)) -> $insPath" -ForegroundColor White - $r = Invoke-FTPSUpload $config.ftps $insFiles $insPath $entityName $pracName - $insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail - } - if ($ptStmts.Count -gt 0) { - Write-Host " PT STMT ($($ptStmts.Count)) -> $pdfPath" -ForegroundColor White - $r = Invoke-FTPSUpload $config.ftps $ptStmts $pdfPath $entityName $pracName - $insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail - } + $totalUploaded = $insFiles.Count + $ptStmts.Count + if ($insResult.Fail -eq 0 -and $totalUploaded -gt 0) { + $archive = Get-ArchiveDir $practice.FullName "Archive sent to FTP" + Move-Item -Path $dateDir -Destination (Join-Path $archive $today) -Force + Write-Host " $today\ archived." -ForegroundColor Green + Write-Info "ARCHIVE entity=$entityName practice=$pracName folder=$today" + } elseif ($insResult.Fail -gt 0) { + Write-Host " $($insResult.Fail) failed - $today\ left for retry." -ForegroundColor Yellow + } elseif ($totalUploaded -eq 0) { + Write-Host " No .$insuranceExt files in $today\." -ForegroundColor Gray + Write-Info "SKIP entity=$entityName practice=$pracName reason=no .$insuranceExt files in date folder" + } - $totalUploaded = $insFiles.Count + $ptStmts.Count - if ($insResult.Fail -eq 0 -and $totalUploaded -gt 0) { - $archive = Get-ArchiveDir $practice.FullName "Archive sent to FTP" - Move-Item -Path $dateDir -Destination (Join-Path $archive $today) -Force - Write-Host " $today\ archived." -ForegroundColor Green - Write-Info "ARCHIVE entity=$entityName practice=$pracName folder=$today" - } elseif ($insResult.Fail -gt 0) { - Write-Host " $($insResult.Fail) failed - $today\ left for retry." -ForegroundColor Yellow - } elseif ($totalUploaded -eq 0) { - Write-Host " No .$insuranceExt files in $today\." -ForegroundColor Gray - Write-Info "SKIP entity=$entityName practice=$pracName reason=no .$insuranceExt files in date folder" + $totalOk += $insResult.Ok; $totalFail += $insResult.Fail } - - $totalOk += $insResult.Ok; $totalFail += $insResult.Fail } # -- Patient files -> SFTP ------------------------------------------------- @@ -326,11 +355,8 @@ foreach ($practice in $practices) { Write-Host " No txt/tif files in $($ptFolder.Name)\ - skipping." -ForegroundColor Gray Write-Info "SKIP entity=$entityName practice=$pracName reason=no txt/tif files" } elseif (($sftpMissing = Get-MissingFields $config.sftp "sftp" @("host", "username", "password", "patient_path")).Count -gt 0) { - Write-Host " CONFIG ERROR: entity.json is missing or has blank: $($sftpMissing -join ', ')" -ForegroundColor Red - Write-Host " Check spelling of those keys in entity.json (and practice.json if present)." -ForegroundColor Yellow - Write-Log "error" "CONFIG entity=$entityName practice=$pracName missing=$($sftpMissing -join ',')" - $totalFail += $patFiles.Count - Write-Host " $($ptFolder.Name)\ left for retry." -ForegroundColor Yellow + Write-Host " Patient upload not configured ($($sftpMissing -join ', ') is null/blank) - $($patFiles.Count) file(s) ignored, left in place." -ForegroundColor Yellow + Write-Info "IGNORE entity=$entityName practice=$pracName reason=unconfigured fields=$($sftpMissing -join ',')" } else { $patPath = Expand-Path $config.sftp.patient_path $pracName Write-Host " Patient ($($patFiles.Count)) -> $patPath" -ForegroundColor White