Ignore overwrite subtable warning for summary rows for old data… (matomo-org#17891)

diosmosis · web-flow · commit 962442c4e4e8 · 2021-08-19T17:24:56.000-07:00
* Ignore warning for summary rows for old data to avoid re-archiving for a single row most users do not look at.

* make sure subtables have ts_archived metadata

* use period start date instead of ts_archived

* try to prevent random failure
diff --git a/core/Archive/DataTableFactory.php b/core/Archive/DataTableFactory.php
@@ -422,6 +422,7 @@ private function setSubtables($dataTable, $blobRow, $treeLevel = 0)
                 $subtable->setMetadata(self::TABLE_METADATA_SITE_INDEX, $dataTable->getMetadata(self::TABLE_METADATA_SITE_INDEX));
                 $subtable->setMetadata(self::TABLE_METADATA_SEGMENT_INDEX, $dataTable->getMetadata(self::TABLE_METADATA_SEGMENT_INDEX));
                 $subtable->setMetadata(self::TABLE_METADATA_SEGMENT_PRETTY_INDEX, $dataTable->getMetadata(self::TABLE_METADATA_SEGMENT_PRETTY_INDEX));
+                $subtable->setMetadata(DataTable::ARCHIVED_DATE_METADATA_NAME, $dataTable->getMetadata(DataTable::ARCHIVED_DATE_METADATA_NAME));
 
                 $this->setSubtables($subtable, $blobRow, $treeLevel + 1);
 
diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php
@@ -11,8 +11,10 @@
 use Exception;
 use Piwik\Container\StaticContainer;
 use Piwik\DataTable;
+use Piwik\Date;
 use Piwik\Log;
 use Piwik\Metrics;
+use Piwik\Period;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -292,7 +294,7 @@ public function sumSubtable(DataTable $subTable)
         if ($this->isSubtableLoaded) {
             $thisSubTable = $this->getSubtable();
         } else {
-            $this->warnIfSubtableAlreadyExists();
+            $this->warnIfSubtableAlreadyExists($subTable);
 
             $thisSubTable = new DataTable();
             $this->setSubtable($thisSubTable);
@@ -765,16 +767,33 @@ public static function isEqual(Row $row1, Row $row2)
         return true;
     }
 
-    private function warnIfSubtableAlreadyExists()
+    private function warnIfSubtableAlreadyExists(DataTable $subTable)
     {
         if (!is_null($this->subtableId)) {
-            $ex = new \Exception(sprintf(
-                "Row with label '%s' (columns = %s) has already a subtable id=%s but it was not loaded - overwriting the existing sub-table.",
-                $this->getColumn('label'),
-                implode(", ", $this->getColumns()),
-                $this->getIdSubDataTable()
-            ));
-            StaticContainer::get(LoggerInterface::class)->warning("{exception}", ['exception' => $ex]);
+            // we only print this warning out if the row isn't a summary row, and if the period start date of the
+            // data is later than the deploy date to cloud for Matomo 4.4.1.
+            //
+            // In 4.4.1 two bugs surrounding the serialization of summary rows with subtables were fixed. Previously,
+            // if a summary row had a subtable when it was inserted into the archive table, this warning would eventually
+            // get triggered. To properly fix this corrupt data, we'd want to invalidate and reporcess it, BUT, that would
+            // require a lot of compute resources, just for the subtable of a row most people would not look at.
+            //
+            // So instead, we simply ignore this issue for data that is for periods older than the deploy date for 4.4.1. If a user
+            // wants to see this subtable data, they can invalidate a specific date and reprocess it. For newer data,
+            // since the bugs were fixed, we don't expect to see the issue. So if the warning gets triggered in this case,
+            // we log the warning in order to be notified.
+            $period = $subTable->getMetadata('period');
+            if (!$this->isSummaryRow()
+                || $this->isStartDateLaterThanCloud441DeployDate($period)
+            ) {
+                $ex = new \Exception(sprintf(
+                    "Row with label '%s' (columns = %s) has already a subtable id=%s but it was not loaded - overwriting the existing sub-table.",
+                    $this->getColumn('label'),
+                    implode(", ", $this->getColumns()),
+                    $this->getIdSubDataTable()
+                ));
+                StaticContainer::get(LoggerInterface::class)->warning("{exception}", ['exception' => $ex]);
+            }
         }
     }
 
@@ -800,4 +819,18 @@ private function getColumnValueDescriptionForError($value)
         }
         return $result;
     }
+
+    private function isStartDateLaterThanCloud441DeployDate($period)
+    {
+        if (empty($period)
+            || !($period instanceof Period)
+        ) {
+            return true; // sanity check
+        }
+
+        $periodStartDate = $period->getDateStart();
+
+        $cloudDeployDate = Date::factory('2021-08-11 12:00:00'); // 2021-08-12 00:00:00 NZST
+        return $periodStartDate->isLater($cloudDeployDate);
+    }
 }
