-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathsync.py
More file actions
157 lines (136 loc) · 6.61 KB
/
sync.py
File metadata and controls
157 lines (136 loc) · 6.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
import os
from datetime import datetime
from typing import List, Dict
from typing_extensions import cast # type: ignore
from polyapi import http_client
from polyapi.utils import get_auth_headers
from polyapi.config import get_api_key_and_url
from polyapi.parser import get_jsonschema_type
from polyapi.deployables import (
prepare_deployable_directory, load_deployable_records,
save_deployable_records, remove_deployable_records,
get_cache_deployments_revision, write_updated_deployable,
DeployableRecord, SyncDeployment, Deployment
)
DEPLOY_ORDER = [
'server-function',
'client-function',
]
def read_file(file_path: str) -> str:
with open(file_path, 'r', encoding='utf-8') as file:
return file.read()
def group_by(items: List[Dict], key: str) -> Dict[str, List[Dict]]:
grouped = {} # type: ignore
for item in items:
grouped.setdefault(item[key], []).append(item)
return grouped
def remove_deployable_function(deployable: SyncDeployment) -> bool:
api_key, _ = get_api_key_and_url()
if not api_key:
raise Exception("Missing api key!")
headers = get_auth_headers(api_key)
url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}/{deployable["id"]}'
response = http_client.get(url, headers=headers)
if response.status_code != 200:
return False
http_client.delete(url, headers=headers)
return True
def remove_deployable(deployable: SyncDeployment) -> bool:
if deployable["type"] == 'client-function' or deployable["type"] == 'server-function':
return remove_deployable_function(deployable)
raise Exception(f"Unsupported deployable type '{deployable['type']}'")
def sync_function_and_get_id(deployable: SyncDeployment, code: str) -> str:
api_key, _ = get_api_key_and_url()
if not api_key:
raise Error("Missing api key!")
headers = get_auth_headers(api_key)
url = f'{deployable["instance"]}/functions/{deployable["type"].replace("-function", "")}'
payload = {
**deployable["config"],
"context": deployable["context"],
"name": deployable["name"],
"description": deployable["description"],
"code": code,
"language": "python",
"returnType": get_jsonschema_type(deployable["types"]["returns"]["type"]),
"returnTypeSchema": deployable["types"]["returns"]["typeSchema"],
"arguments": [{**p, "key": p["name"], "type": get_jsonschema_type(p["type"]) } for p in deployable["types"]["params"]],
}
response = http_client.post(url, headers=headers, json=payload)
response.raise_for_status()
return response.json()['id']
def sync_deployable_and_get_id(deployable: SyncDeployment, code: str) -> str:
if deployable["type"] == 'client-function' or deployable["type"] == 'server-function':
return sync_function_and_get_id(deployable, code)
raise Exception(f"Unsupported deployable type '{deployable['type']}'")
def sync_deployable(deployable: SyncDeployment) -> Deployment:
code = read_file(deployable['file'])
id = sync_deployable_and_get_id(deployable, code)
return {
"name": deployable["name"],
"context": deployable["context"],
"instance": deployable["instance"],
"type": deployable["type"],
"id": id,
"deployed": datetime.now().isoformat(),
"fileRevision": deployable["fileRevision"],
}
def sync_deployables(dry_run: bool, instance: str | None = None):
if not instance:
_, instance = get_api_key_and_url()
prepare_deployable_directory()
git_revision = get_cache_deployments_revision()
all_deployables = load_deployable_records()
to_remove: List[DeployableRecord] = []
if not all_deployables:
print('No deployables found. Skipping sync.')
return
# TODO: Improve our deploy ordering.
# Right now we're doing rudimentary ordering by type
# But this does not safely handle cases where one server function may reference another
# We should parse the functions bodies for references to other Poly deployables and work them into a DAG
grouped_deployables = group_by(all_deployables, 'type')
for type_name in DEPLOY_ORDER:
deployables = grouped_deployables.get(type_name, [])
for deployable in deployables:
previous_deployment = None
try:
previous_deployment = next((d for d in deployable.get('deployments', []) if d['instance'] == instance), None)
except:
pass
git_revision_changed = git_revision != deployable['gitRevision']
file_revision_changed = not previous_deployment or previous_deployment['fileRevision'] != deployable['fileRevision']
action = 'REMOVED' if git_revision_changed else \
'ADDED' if not previous_deployment else \
'UPDATED' if file_revision_changed else 'OK'
if not dry_run and (git_revision_changed or file_revision_changed):
# Any deployable may be deployed to multiple instances/environments at the same time
# So we reduce the deployable record down to a single instance we want to deploy to
if previous_deployment:
sync_deployment = {
**deployable,
**previous_deployment,
"description": deployable["types"]["description"],
"instance": instance
}
else:
sync_deployment = { **deployable, "instance": instance }
if git_revision == deployable['gitRevision']:
deployment = sync_deployable(cast(SyncDeployment, sync_deployment))
if previous_deployment:
previous_deployment.update(deployment)
else:
deployable['deployments'].insert(0, deployment)
else:
found = remove_deployable(cast(SyncDeployment, sync_deployment))
action = 'NOT FOUND' if not found else action
remove_index = all_deployables.index(cast(DeployableRecord, deployable))
to_remove.append(all_deployables.pop(remove_index))
print(f"{'Would sync' if dry_run else 'Synced'} {deployable['type'].replace('-', ' ')} {deployable['context']}.{deployable['name']}: {'TO BE ' if dry_run else ''}{action}")
if dry_run:
return
for deployable in all_deployables:
write_updated_deployable(deployable, True)
save_deployable_records(all_deployables)
if to_remove:
remove_deployable_records(to_remove)