NRL-2004 Migrate v1 permissions script

NHSDigital · sandyforresternhs · Mar 25, 2026 · Mar 23, 2026 · Mar 23, 2026 · Mar 25, 2026
commit de768db67bebd38d33dbc8f6443cf4be6fe3aeaf
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""
+Reads JSON files from a given source folder in the environment's S3 authorization
+bucket, transforms each from a flat array into {"types": [...]} format, and
+writes the results to both the consumer and producer folders in the same bucket
+under a sub-folder matching the source folder name.
+
+Usage:
+    python scripts/migrate_v1_perms_by_app.py <env> <folder>
+
+Arguments:
+    env     - NRLF environment name (e.g. dev, qa, int, prod)
+    folder  - Source folder name within the authorization bucket
+              (e.g. an app identifier)
+
+Example:
+    python scripts/migrate_v1_perms_by_app.py dev my-app-folder
+
+The script reads from:
+    s3://nhsd-nrlf--<env>-authorization-store/<folder>/*.json
+
+And writes to:
+    s3://nhsd-nrlf--<env>-authorization-store/consumer/<folder>/<filename>.json
+    s3://nhsd-nrlf--<env>-authorization-store/producer/<folder>/<filename>.json
+
+The bucket name defaults to nhsd-nrlf--<env>-authorization-store and can be
+overridden via the NRL_AUTH_BUCKET_NAME environment variable.
+"""
+
+import json
+import os
+import sys
+
+from aws_session_assume import get_boto_session
+from botocore.exceptions import ClientError
+
+CONSUMER_OR_PRODUCER = ("consumer", "producer")
+
+
+def _get_bucket_name(env: str) -> str:
+    return os.getenv("NRL_AUTH_BUCKET_NAME", f"nhsd-nrlf--{env}-authorization-store")
+
+
+def _get_s3_client(env: str):
+    return get_boto_session(env).client("s3")
+
+
+def _list_json_files(s3, bucket: str, folder: str) -> list[str]:
+    paginator = s3.get_paginator("list_objects_v2")
+    return sorted(
+        item["Key"]
+        for page in paginator.paginate(Bucket=bucket, Prefix=f"{folder}/")
+        for item in page.get("Contents", [])
+        if item["Key"].endswith(".json")
+    )
+
+
+def _read_and_transform(s3, bucket: str, file_path: str) -> str:
+    try:
+        response = s3.get_object(Bucket=bucket, Key=file_path)
+    except ClientError as e:
+        raise RuntimeError(
+            f"Failed to read s3://{bucket}/{file_path}: {e.response['Error']['Message']}"
+        ) from e
+    data = json.loads(response["Body"].read())
+
+    if not isinstance(data, list):
+        raise ValueError(
+            f"{file_path}: Expected a JSON array, got {type(data).__name__}"
+        )
+
+    return json.dumps({"types": data}, indent=2), len(data)
+
+
+def _write_v2_consumer_and_producer_files(
+    s3, bucket: str, file_path: str, body: str, entry_count: int
+) -> None:
+    for actor_type in CONSUMER_OR_PRODUCER:
+        dest_filepath = f"{actor_type}/{file_path}"
+        try:
+            s3.put_object(Bucket=bucket, Key=dest_filepath, Body=body)
+        except ClientError as e:
+            raise RuntimeError(
+                f"Failed to write s3://{bucket}/{dest_filepath}: {e.response['Error']['Message']}"
+            ) from e
+        print(f"  Written {entry_count} entries → s3://{bucket}/{dest_filepath}")
+
+
+def migrate_v1_perms_by_app(env: str, app_id_folder: str) -> None:
+    bucket = _get_bucket_name(env)
+    s3 = _get_s3_client(env)
+
+    print(f"Source bucket : {bucket}")
+    print(f"Source folder : {app_id_folder}/")
+
+    json_file_paths = _list_json_files(s3, bucket, app_id_folder)
+    if not json_file_paths:
+        print(f"No JSON files found under s3://{bucket}/{app_id_folder}/")
+        return
+    print(f"Found {len(json_file_paths)} JSON files in s3://{bucket}/{app_id_folder}/:")
+
+    for file_path in json_file_paths:
+        body, entry_count = _read_and_transform(s3, bucket, file_path)
+        print(f"  Transforming {file_path} → {entry_count} entries")
+
+        _write_v2_consumer_and_producer_files(s3, bucket, file_path, body, entry_count)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} <env> <folder>")
+        sys.exit(1)
+
+    migrate_v1_perms_by_app(sys.argv[1], sys.argv[2])
@@ -0,0 +1,162 @@
+import io
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+from botocore.exceptions import ClientError
+from migrate_v1_perms_by_app import (
+    CONSUMER_OR_PRODUCER,
+    _read_and_transform,
+    migrate_v1_perms_by_app,
+)
+
+ENV = "dev"
+FOLDER = "my-app-folder"
+BUCKET = f"nhsd-nrlf--{ENV}-authorization-store"
+
+SAMPLE_V1_PERMS = [
+    "http://snomed.info/sct|736253002",
+    "http://snomed.info/sct|1363501000000100",
+    "http://snomed.info/sct|736366004",
+]
+
+# ---------------------------------------------------------------------------
+# Helper functions
+# ---------------------------------------------------------------------------
+
+
+def _make_client_error(code: str, message: str) -> ClientError:
+    return ClientError(
+        {"Error": {"Code": code, "Message": message}},
+        operation_name="S3Operation",
+    )
+
+
+def _mock_s3_client_with_response(data_to_return: bytes) -> MagicMock:
+    s3 = MagicMock()
+    s3.get_object.return_value = {"Body": io.BytesIO(data_to_return)}
+    return s3
+
+
+# ---------------------------------------------------------------------------
+# Unit tests for _read_and_transform
+# ---------------------------------------------------------------------------
+
+FILE_PATH = f"{FOLDER}/perms.json"
+
+
+def test_read_and_transform_returns_wrapped_json_and_count():
+    s3 = _mock_s3_client_with_response(json.dumps(SAMPLE_V1_PERMS).encode())
+
+    body, count = _read_and_transform(s3, BUCKET, FILE_PATH)
+
+    assert count == len(SAMPLE_V1_PERMS)
+    assert json.loads(body) == {"types": SAMPLE_V1_PERMS}
+    s3.get_object.assert_called_once_with(Bucket=BUCKET, Key=FILE_PATH)
+
+
+def test_read_and_transform_empty_list():
+    s3 = _mock_s3_client_with_response(b"[]")
+
+    body, count = _read_and_transform(s3, BUCKET, FILE_PATH)
+
+    assert count == 0
+    assert json.loads(body) == {"types": []}
+
+
+def test_read_and_transform_raises_value_error_for_non_list():
+    s3 = _mock_s3_client_with_response(b'{"key": "value"}')
+
+    with pytest.raises(ValueError, match="Expected a JSON array, got dict"):
+        _read_and_transform(s3, BUCKET, FILE_PATH)
+
+
+def test_read_and_transform_raises_runtime_error_on_client_error():
+    s3 = MagicMock()
+    s3.get_object.side_effect = _make_client_error(
+        "NoSuchKey", "The specified key does not exist"
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match=f"Failed to read s3://{BUCKET}/{FILE_PATH}.*The specified key does not exist",
+    ):
+        _read_and_transform(s3, BUCKET, FILE_PATH)
+
+
+# ---------------------------------------------------------------------------
+# Unit tests for migrate_v1_perms_by_app
+# ---------------------------------------------------------------------------
+
+MODULE = "migrate_v1_perms_by_app"
+
+TRANSFORMED_BODY = '{"types": ["http://snomed.info/sct|736253002"]}'
+ENTRY_COUNT = 1
+
+
+@patch(f"{MODULE}._write_v2_consumer_and_producer_files")
+@patch(f"{MODULE}._read_and_transform")
+@patch(f"{MODULE}._list_json_files")
+@patch(f"{MODULE}._get_s3_client")
+@patch(f"{MODULE}._get_bucket_name")
+def test_migrate_processes_each_file(
+    mock_bucket, mock_s3, mock_list, mock_transform, mock_write
+):
+    mock_bucket.return_value = BUCKET
+    s3 = MagicMock()
+    mock_s3.return_value = s3
+    mock_list.return_value = [f"{FOLDER}/a.json", f"{FOLDER}/b.json"]
+    mock_transform.return_value = (TRANSFORMED_BODY, ENTRY_COUNT)
+
+    migrate_v1_perms_by_app(ENV, FOLDER)
+
+    mock_bucket.assert_called_once_with(ENV)
+    mock_s3.assert_called_once_with(ENV)
+    mock_list.assert_called_once_with(s3, BUCKET, FOLDER)
+    assert mock_transform.call_count == 2
+    assert mock_write.call_count == 2
+    mock_write.assert_any_call(
+        s3, BUCKET, f"{FOLDER}/a.json", TRANSFORMED_BODY, ENTRY_COUNT
+    )
+    mock_write.assert_any_call(
+        s3, BUCKET, f"{FOLDER}/b.json", TRANSFORMED_BODY, ENTRY_COUNT
+    )
+
+
+@patch(f"{MODULE}._write_v2_consumer_and_producer_files")
+@patch(f"{MODULE}._read_and_transform")
+@patch(f"{MODULE}._list_json_files")
+@patch(f"{MODULE}._get_s3_client")
+@patch(f"{MODULE}._get_bucket_name")
+def test_migrate_no_files_skips_transform_and_write(
+    mock_bucket, mock_s3, mock_list, mock_transform, mock_write
+):
+    mock_bucket.return_value = BUCKET
+    mock_s3.return_value = MagicMock()
+    mock_list.return_value = []
+
+    migrate_v1_perms_by_app(ENV, FOLDER)
+
+    mock_transform.assert_not_called()
+    mock_write.assert_not_called()
+
+
+@patch(f"{MODULE}._write_v2_consumer_and_producer_files")
+@patch(f"{MODULE}._read_and_transform")
+@patch(f"{MODULE}._list_json_files")
+@patch(f"{MODULE}._get_s3_client")
+@patch(f"{MODULE}._get_bucket_name")
+def test_migrate_passes_transform_output_to_write(
+    mock_bucket, mock_s3, mock_list, mock_transform, mock_write
+):
+    mock_bucket.return_value = BUCKET
+    s3 = MagicMock()
+    mock_s3.return_value = s3
+    mock_list.return_value = [f"{FOLDER}/perms.json"]
+    mock_transform.return_value = (TRANSFORMED_BODY, ENTRY_COUNT)
+
+    migrate_v1_perms_by_app(ENV, FOLDER)
+
+    mock_write.assert_called_once_with(
+        s3, BUCKET, f"{FOLDER}/perms.json", TRANSFORMED_BODY, ENTRY_COUNT
+    )