Skip to content

Commit 1d64b69

Browse files
SimonGoringSimonGoring
committed
A lot of changes. All pushing.
1 parent c15fe4d commit 1d64b69

21 files changed

+452
-16
lines changed

‎.gitignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,15 @@ ignore/*
66
*.R
77

88
helpers/dumps/
9+
10+
.vscode/settings.json
11+
12+
helpers/dumps2021-06-07/
13+
14+
helpers/figshareUpload/lib/__pycache__/
15+
16+
helpers/figshareUpload/settings.yaml
17+
18+
helpers/settings.yaml
19+
20+
fixGeorge.sh

‎bash/newSteward.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ display_help() {
1818
echo " -t taxonomyexpert Is the steward a taxonomy expert? [true|false]"
1919
echo " -d database Which database is the steward associate with?"
2020
echo
21+
echo " The script assumes that you have a ~/.pgpass file that contains your password"
22+
echo " information. The file is described in detail on the Postgres documentation:"
23+
echo " https://www.postgresql.org/docs/current/libpq-pgpass.html"
24+
echo
2125
# echo some stuff here for the -a or --add-options
2226
exit 1
2327
}

‎function/da/geopoliticalunits.sql‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
CREATE MATERIALIZED VIEW da.geopoliticalunits AS
2+
SELECT DISTINCT sgp.geopoliticalid,
3+
gpu.geopoliticalname,
4+
gpu.rank,
5+
gpu.highergeopoliticalid
6+
FROM ndb.sitegeopolitical sgp
7+
JOIN ndb.geopoliticalunits gpu ON gpu.geopoliticalid = sgp.geopoliticalid
8+
ORDER BY gpu.geopoliticalname;
9+
10+
CREATE INDEX geopolrank ON da.geopoliticalunits USING hash(rank);
11+
12+
CREATE OR REPLACE FUNCTION da.refresh_sitegeopol()
13+
RETURNS TRIGGER LANGUAGE plpgsql
14+
AS $$
15+
BEGIN
16+
REFRESH MATERIALIZED VIEW CONCURRENTLY da.geopoliticalunits;
17+
RETURN NULL;
18+
END $$;
19+
20+
CREATE TRIGGER da_geopol_trigger
21+
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE
22+
ON ndb.sitegeopolitical
23+
FOR EACH STATEMENT
24+
EXECUTE PROCEDURE da.refresh_sitegeopol();

‎function/doi/02_ndbdata.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ WITH dssamples AS (
5858
)
5959
SELECT
6060
ds.datasetid,
61-
jsonb_build_object('dataset', dsinfo.dataset,
62-
'samples', json_agg(dss.sampledata)) AS data
61+
dsinfo.dataset,
62+
jsonb_build_object('samples', json_agg(dss.sampledata)) AS data
6363
FROM
6464
ndb.datasets AS ds
6565
JOIN dssamples AS dss ON ds.datasetid = dss.datasetid

‎function/indexes/addingIndices.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,11 @@ CREATE INDEX IF NOT EXISTS smpageyoung_idx ON ndb.sampleages USING btree(ageyoun
1818
CREATE INDEX IF NOT EXISTS geoage_idx ON ndb.geochronology USING btree(age);
1919

2020
CREATE INDEX IF NOT EXISTS variableel ON ndb.variables USING btree(taxonid, variableelementid, variableunitsid);
21+
22+
CREATE INDEX IF NOT EXISTS data_variable_idx ON ndb.data USING btree(variableid);
23+
CREATE INDEX IF NOT EXISTS data_sample_idx ON ndb.data USING btree(sampleid);
24+
25+
CREATE INDEX IF NOT EXISTS sample_taxon_idx ON ndb.samples USING btree(taxonid);
26+
27+
CREATE INDEX IF NOT EXISTS analysisunits_collunit_idx ON ndb.analysisunits USING btree(collectionunitid);
28+
CREATE INDEX IF NOT EXISTS analysisunits_facies_idx ON ndb.analysisunits USING btree(faciesid);

function/longgeoname_view.SQL renamed to function/longgeoname_view.sql

File renamed without changes.

‎helpers/figshareUpload/figshare_upload.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import hashlib
2+
import json
3+
import requests
4+
from requests.exceptions import HTTPError
5+
import time
6+
import yaml
7+
from lib.pgBackup import backup_database
8+
import lib.figShare as fs
9+
10+
# The following two functions are drawn from this code on StackOverflow:
11+
# https://stackoverflow.com/a/39999652/14302148
12+
13+
token = '<insert access token here>'
14+
FILE_PATH = '/path/to/work/directory/cat.obj'
15+
16+
TITLE = 'A 3D cat object model'
17+
18+
article_id = fs.create_article(TITLE, token)
19+
20+
# Then we upload the file.
21+
file_info = fs.initiate_new_upload(article_id, FILE_PATH, token)
22+
23+
# Until here we used the figshare API; following lines use the figshare upload service API.
24+
fs.upload_parts(file_info=file_info, token=token, file_path=FILE_PATH)
25+
26+
# We return to the figshare API to complete the file upload process.
27+
fs.complete_upload(article_id, file_info['id'])

‎helpers/figshareUpload/lib/__init__.py‎

Whitespace-only changes.

‎helpers/figshareUpload/lib/figShare.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import hashlib
2+
import json
3+
import os
4+
import requests
5+
from requests.exceptions import HTTPError
6+
7+
def raw_issue_request(method, url, token, data=None, binary=False):
8+
headers = {'Authorization': 'token ' + token}
9+
if data is not None and not binary:
10+
data = json.dumps(data)
11+
response = requests.request(method, url, headers=headers, data=data)
12+
try:
13+
response.raise_for_status()
14+
try:
15+
data = json.loads(response.content)
16+
except ValueError:
17+
data = response.content
18+
except HTTPError as error:
19+
print('Caught an HTTPError: %s', str(error))
20+
print ('Body:\n', response.content)
21+
raise
22+
23+
return data
24+
25+
26+
def issue_request(method, endpoint,
27+
*args, **kwargs):
28+
baseurl='https://api.figshare.com/v2/{endpoint}'
29+
return raw_issue_request(method, baseurl.format(endpoint=endpoint), *args, **kwargs)
30+
31+
32+
def create_article(title, token):
33+
data = {
34+
'title': title # You may add any other information about the article here as you wish.
35+
}
36+
result = issue_request('POST', 'account/articles', data=data)
37+
print('Created article:', result['location'], '\n')
38+
39+
result = raw_issue_request('GET', result['location'], token)
40+
41+
return result['id']
42+
43+
44+
def list_files_of_article(article_id):
45+
result = issue_request('GET', 'account/articles/{}/files'.format(article_id))
46+
print('Listing files for article {}:'.format(article_id))
47+
if result:
48+
for item in result:
49+
print(' {id} - {name}'.format(**item))
50+
else:
51+
print(' No files.')
52+
53+
print
54+
55+
56+
def get_file_check_data(file_name):
57+
chunkSize = 1048576
58+
with open(file_name, 'rb') as fin:
59+
md5 = hashlib.md5()
60+
size = 0
61+
data = fin.read(chunkSize)
62+
while data:
63+
size += len(data)
64+
md5.update(data)
65+
data = fin.read(chunkSize)
66+
return md5.hexdigest(), size
67+
68+
69+
def initiate_new_upload(article_id, file_name, token):
70+
endpoint = 'account/articles/{}/files'
71+
endpoint = endpoint.format(article_id)
72+
73+
md5, size = get_file_check_data(file_name)
74+
data = {'name': os.path.basename(file_name),
75+
'md5': md5,
76+
'size': size}
77+
78+
result = issue_request('POST', endpoint, data=data)
79+
print('Initiated file upload:', result['location'], '\n')
80+
81+
result = raw_issue_request('GET', result['location'], token)
82+
83+
return result
84+
85+
86+
def complete_upload(article_id, file_id):
87+
issue_request('POST', 'account/articles/{}/files/{}'.format(article_id, file_id))
88+
89+
90+
def upload_parts(file_info, token, file_path):
91+
url = '{upload_url}'.format(**file_info)
92+
result = raw_issue_request('GET', url, token)
93+
94+
print('Uploading parts:')
95+
with open(file_path, 'rb') as fin:
96+
for part in result['parts']:
97+
upload_part(file_info, fin, part, token)
98+
print
99+
100+
101+
def upload_part(file_info, stream, part, token):
102+
udata = file_info.copy()
103+
udata.update(part)
104+
url = '{upload_url}/{partNo}'.format(**udata)
105+
106+
stream.seek(part['startOffset'])
107+
data = stream.read(part['endOffset'] - part['startOffset'] + 1)
108+
109+
raw_issue_request('PUT', url, token, data=data, binary=True)
110+
print (' Uploaded part {partNo} from {startOffset} to {endOffset}'.format(**part))

‎helpers/figshareUpload/lib/pgBackup.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import time
2+
import yaml
3+
import os
4+
5+
def create_essentials():
6+
settings = yaml.load(open("settings.yaml", 'r'), yaml.SafeLoader)
7+
db_name = settings["db_name"]
8+
db_user = settings["db_user"]
9+
db_host = settings["db_host"]
10+
db_port = settings["db_port"]
11+
backup_path = settings["backup_path"]
12+
filename = settings["filename"]
13+
filename = filename + "-" + time.strftime("%Y%m%d") + ".backup"
14+
command_str = str(db_host)+" -p "+str(db_port)+" -d "+db_name+" -U "+db_user
15+
return command_str, backup_path, filename
16+
17+
18+
def backup_database(table_names=None):
19+
command_str,backup_path,filename = create_essentials()
20+
command_str = "pg_dump -h " + command_str
21+
22+
if table_names is not None:
23+
for x in table_names:
24+
command_str = command_str +" -t "+x
25+
26+
command_str = command_str + " -F c -b -v -f '"+backup_path+"/"+filename+"'"
27+
try:
28+
os.system(command_str)
29+
print("Backup completed")
30+
except Exception as e:
31+
print("!!Problem occured!!")
32+
print(e)

0 commit comments

Comments
 (0)