Skip to content

Commit d90c32d

Browse files
lucacuglucacug
authored
Merge pull request #983 from unepwcmc/hotfix/1.14.1
Hotfix/1.14.1
2 parents df60bcd + dcf377b commit d90c32d

7 files changed

Lines changed: 68 additions & 7 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
### 1.14.1
2+
3+
**Rails 6 Upgrade**
4+
5+
Hotfixes to solve issues mostly relating to the Rails 6 upgrade
6+
7+
* Sanitizing some parameters.
8+
* Avoiding retries on jobs which was causing instability during the overnight
9+
scripts.
10+
* Fixing an issue with storing timestamped dates which was affected by the
11+
difference between BST and UTC.
12+
113
### 1.14.0
214

315
**Rails 6 Upgrade**

‎app/jobs/application_job.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
11
class ApplicationJob < ActiveJob::Base
2+
# Some jobs are long-running, others are triggered frequently.
3+
# Long-running jobs which fail should not be retried every few hours.
4+
# Default to not retrying. Jobs that are safe to retry can override this.
5+
sidekiq_options retry: false
26
end

‎app/models/trade/inclusion_validation_rule.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ def matching_record_to_matching_hash(mr)
130130
column_names.map do |column_name|
131131
column_value = mr.send(column_name)
132132
if column_value.nil?
133-
[ column_name, column_value ]
133+
[ column_name, "" ]
134134
else
135135
[ column_name, column_value.to_s ]
136136
end

‎config/application.rb‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,13 @@ class Application < Rails::Application
1717
#
1818
# These settings can be overridden in specific environments using the files
1919
# in config/environments, which are processed later.
20+
21+
# Don't set the time zone - it causes issues when converting to columns of
22+
# type TIMESTAMP WITHOUT TIME ZONE - during BST, dates without time parts
23+
# first become midnight, then become 2300 the day before.
2024
#
21-
config.time_zone = "London"
25+
# config.time_zone = "London"
26+
2227
# config.eager_load_paths << Rails.root.join("extras")
2328

2429
# @see https://gist.github.com/maxivak/381f1e964923f1d469c8d39da8e2522f

‎config/deploy/production.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
set :stage, :production
2-
set :branch, "master"
2+
set :branch, ENV['CAP_BRANCH'] || 'master'
33

44
server "sapi-production.linode.unep-wcmc.org", user: "wcmc", roles: %w{app web db}
55

‎lib/modules/material_doc_ids_retriever.rb‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,17 +69,27 @@ def self.locale_document(doc)
6969
end
7070

7171
def self.ancestors_ids(tc_ids, taxon_name = nil, exact_match = nil)
72+
# TODO: Raise an error if any of these are not integers instead of leaving
73+
# it to the db to do this. That logic maybe belongs elsewhere?
74+
# Consider using `SearchParamSanitiser.sanitise_integer_array` for this.
75+
tc_ids_array = Array(tc_ids.is_a?(String) ? tc_ids.split(',') : tc_ids)
76+
tc_ids_sql = tc_ids_array.map(&:to_i).filter{ |tc_id| tc_id > 0 }.join(', ')
77+
78+
# Don't bother running a query if we didn't get any taxon concept ids
79+
return [] if tc_ids_sql.empty?
80+
7281
res = ApplicationRecord.connection.execute(
7382
<<-SQL
7483
SELECT ancestor_taxon_concept_id
7584
FROM taxon_concepts_and_ancestors_mview
76-
WHERE taxon_concept_id IN (#{tc_ids})
85+
WHERE taxon_concept_id IN (#{tc_ids_sql})
7786
AND ancestor_taxon_concept_id IS NOT NULL
7887
ORDER BY
7988
#{order_case(exact_match, taxon_name)}
8089
tree_distance DESC, ancestor_taxon_concept_id;
8190
SQL
8291
)
92+
8393
res.map(&:values).flatten.map(&:to_i).uniq
8494
end
8595

‎lib/modules/sapi_module/stored_procedures.rb‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
module SapiModule
22
module StoredProcedures
33

4+
##
5+
# This takes several hours to run:
6+
#
7+
# - The first hour is for all the views prior to trade_plus_complete view
8+
# - The remainder of the time is exclusively spent on trade_plus_complete_view
9+
#
10+
# Runtime grows as the database grows, and in particular the trade plus
11+
# dataset is growing quite a bit year on year.
12+
413
def self.rebuild
514
ActiveRecord::Base.transaction do
6-
[
15+
# The names of the functions beginnning 'rebuild_' to be run in sequence
16+
to_rebuild = [
717
:taxonomy,
818
:cites_accepted_flags,
919
:listing_changes_mview,
@@ -24,9 +34,29 @@ def self.rebuild
2434
:trade_shipments_cites_suspensions_mview,
2535
:non_compliant_shipments_view,
2636
:trade_plus_complete_mview
27-
].each { |p|
37+
]
38+
39+
connection = ActiveRecord::Base.connection
40+
41+
to_lock = connection.execute(
42+
# This is not great, because it relies on things being called mview
43+
# when they're not matviews, it's the tables we're locking, matviews
44+
# don't respond to LOCK TABLE.
45+
"SELECT relname FROM pg_class WHERE relname LIKE '%_mview' AND relkind = 'r';"
46+
).to_a.map{ |row| row['relname'] }
47+
48+
to_lock.each { |relname|
49+
# Lock tables in advance to prevent deadlocks forcing a rollback.
50+
puts "Locking table: #{relname}"
51+
52+
# We need ACCESS EXCLUSIVE because this is used by DROP TABLE, and
53+
# most of the rebuild_... functions are dropping and recreating the
54+
# matviews.
55+
connection.execute("LOCK TABLE #{relname} IN ACCESS EXCLUSIVE MODE")
56+
}
57+
58+
to_rebuild.each { |p|
2859
puts "Procedure: #{p}"
29-
connection = ActiveRecord::Base.connection
3060

3161
# Within the current transaction, set work_mem to a higher-than-usual
3262
# value, so that matviews can be built more efficiently.

0 commit comments

Comments
 (0)