-
Notifications
You must be signed in to change notification settings - Fork 22
Expand file tree
/
Copy pathfuns.py
More file actions
45 lines (34 loc) · 1.21 KB
/
funs.py
File metadata and controls
45 lines (34 loc) · 1.21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import pandas as pd
import json
import os
import sys
import datetime
import time
import string
def clean_json(item_json):
valid_chars = string.ascii_letters + string.digits + "_"
def replace(key):
return ''.join([x if x in valid_chars else '_' for x in key]).lower()
def valid_key(key):
return key == replace(key)
for key in item_json:
if not valid_key(key):
item_json[replace(key)] = item_json[key]
item_json.pop(key, None)
return item_json
def get_timestamp():
return datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S.%f')
def save_json(response_json, fileName, args):
with open(fileName, 'w') as file:
for item_json in response_json:
if args.timestamp:
item_json['_timestamp'] = get_timestamp()
if args.clean:
item_json = clean_json(item_json)
json.dump(item_json, file)
file.write(os.linesep)
def save_csv(response_json, fileName, args):
df = pd.DataFrame(response_json)
if args.timestamp:
df = df.assign(_timestamp = timestamp_str)
df.to_csv(fileName, index=False, header=not args.header, decimal='.', sep= args.sep, encoding='utf-8')