55_DWD_OpenData.pm: fix forecast rotation at daybreak (forum #83097 msg #1108423)

git-svn-id: https://svn.fhem.de/fhem/trunk@23392 2b470e98-0d58-463d-a4d8-8e2adae1ed80

fhem/CHANGED

@@ -1,11 +1,12 @@
 # Add changes at the top of the list. Keep it in ASCII, and 80-char wide.
 # Do not insert empty lines here, update check depends on it.
+ - bugfix:  55_DWD_OpenData: forecast rotation (forum #83097 msg #1108423)
  - feature: 02_RSS: added readings to show filename and type of background
  - bugfix:  57_SSCal: accept calendar names which are contain umlauts
  - feature: 98_weekprofile: add support for MQTT2_DEVICE (txl Beta-User)
  - feature: 89_FULLY: Added support for sensor information
  - bugfix: 98_weekprofile: HMCCU-HM send profile without prefix (Forum #1104569)
- - bugfix: 70_DENON_AVR: serial connection issues
+ - bugfix:  70_DENON_AVR: serial connection issues
  - feature: 77_SMAEM: attr serialNumber may contain multiple serial numbers
  - feature: 57_SSCal: handle multiline code in description field if automatic
                       creation of at-devices is used

fhem/FHEM/55_DWD_OpenData.pm

@@ -35,23 +35,24 @@ Julian date conversion:
   Copyright (C) 2012 E. G. Richards
     see Explanatory Supplement to the Astronomical Almanac, 3rd edition, S.E Urban and P.K. Seidelmann eds., chapter 15.11.3, Interconverting Dates and Julian Day Numbers, Algorithm 4
 
-This script is free software; you can redistribute it and/or modify
+This script is free software; you can redistribute it and/or
-it under the terms of the GNU General Public License as published by
+modify it under the terms of the GNU General Public License
-the Free Software Foundation; either version 2 of the License, or
+as published by the Free Software Foundation; either version 2
-(at your option) any later version.
+of the License, or (at your option) any later version.
-
-The GNU General Public License can be found at
-
-http://www.gnu.org/copyleft/gpl.html.
-
-A copy is found in the textfile GPL.txt and important notices to the license
-from the author is found in LICENSE.txt distributed with these scripts.
 
 This script is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 GNU General Public License for more details.
 
+You should have received a copy of the GNU General Public License
+along with this script; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+A copy of the GNU General Public License, Version 2 can also be found at
+
+http://www.gnu.org/licenses/old-licenses/gpl-2.0.
+
 This copyright notice MUST APPEAR in all copies of the script!
 
 =cut
@@ -616,7 +617,7 @@ use constant UPDATE_COMMUNEUNIONS => -2;
 use constant UPDATE_ALL           => -3;
 
 require Exporter;
-our $VERSION   = '1.014006';
+our $VERSION   = '1.016001';
 our @ISA       = qw(Exporter);
 our @EXPORT    = qw(GetForecast GetAlerts UpdateAlerts UPDATE_DISTRICTS UPDATE_COMMUNEUNIONS UPDATE_ALL);
 our @EXPORT_OK = qw(IsCommuneUnionWarncellId);
@@ -850,7 +851,7 @@ sub Shutdown {
       delete($hash->{".alertsFile".$communeUnion});
     }
   }
-  
+
   if (defined($hash->{".forecastBlockingCall"})) {
     ::BlockingKill($hash->{".forecastBlockingCall"});
   }
@@ -1399,7 +1400,7 @@ sub RotateForecast {
   while (defined(::ReadingsVal($name, 'fc'.$daysAvailable.'_date', undef))) {
     $daysAvailable++;
   }
-  ::Log3 $name, 5, "$name: RotateForecast: $daysAvailable days exist with readings";
+  ::Log3 $name, 5, "$name: RotateForecast: START $daysAvailable day(s) exist";
 
   my $oT = ::ReadingsVal($name, 'fc0_date', undef);
   my $oldToday = defined($oT)? ParseDateLocal($hash, $oT) : undef;
@@ -1407,6 +1408,7 @@ sub RotateForecast {
   my $stationChanged = ::ReadingsVal($name, 'fc_station', '') ne $station;
   if ($stationChanged) {
     # different station, delete all existing readings
+    ::Log3 $name, 3, "$name: RotateForecast: station has changed, deleting exisiting readings";
     ::CommandDeleteReading(undef, "$name ^fc.*");
     $daysAvailable = 0;
   } elsif (defined($oldToday)) {
@@ -1418,7 +1420,7 @@ sub RotateForecast {
     }
 
     my $daysForward = sprintf("%0.0f", ($today - $oldToday)/86400.0);  # round() [s] -> [d]
-    ::Log3 $name, 5, "$name: RotateForecast: shifting forward by $daysForward day(s) ($oldToday -> $today)";
+    ::Log3 $name, 3, "$name: RotateForecast: shifting forward by $daysForward day(s) ($oldToday -> $today)";
     if ($daysForward > 0) {
       # different day
       if ($daysForward < $daysAvailable) {
@@ -1437,16 +1439,24 @@ sub RotateForecast {
           push(@shiftProperties, $s.'_time');
           push(@shiftProperties, $s.'_wwd');
         }
-        # shift readings forward by days
+        # shift readings forward by days keeping reading timestamps
         for (my $d=0; $d<($daysAvailable - $daysForward); $d++) {
           my $sourcePrefix = 'fc'.($daysForward + $d).'_';
           my $destinationPrefix = 'fc'.$d.'_';
           foreach my $property (@shiftProperties) {
-            my $value = ::ReadingsVal($name, $sourcePrefix.$property, undef);
+            my $sourceReading = $sourcePrefix.$property;
-            if (defined($value)) {
+            my $destinationReading = $destinationPrefix.$property;
-              ::readingsBulkUpdate($hash, $destinationPrefix.$property, $value);
+            my $sourceValue = ::ReadingsVal($name, $sourceReading, undef);
+            if (defined($sourceValue)) {
+              my $timestamp = $hash->{READINGS}{$sourceReading}{TIME};
+              ::readingsBulkUpdate($hash, $destinationReading, $sourceValue);
+              $hash->{READINGS}{$destinationReading}{TIME} = $timestamp;
             } else {
-              ::CommandDeleteReading(undef, $destinationPrefix.$property);
+              my $destinationValue = ::ReadingsVal($name, $destinationReading, undef);
+              if (defined($destinationValue)) {
+                ::Log3 $name, 3, "$name: RotateForecast WARNING: deleting reading $destinationReading because the source value $sourceReading is undefined";
+                ::CommandDeleteReading(undef, "$name $destinationReading");
+              }
             }
           }
         }
@@ -1457,15 +1467,78 @@ sub RotateForecast {
         $daysAvailable -= $daysForward;
       } else {
         # nothing remains after shifting, delete existing day readings
+        ::Log3 $name, 3, "$name: RotateForecast WARNING: deleting all readings because no forecast data remains for rotation";
         ::CommandDeleteReading(undef, "$name ^fc\\d+.*");
         $daysAvailable = 0;
       }
     }
   }
 
+  ::Log3 $name, 5, "$name: RotateForecast: END $daysAvailable day(s) remain";
+
   return $daysAvailable;
 }
 
+=head2 PruneForecast($)
+
+find youngest reading of each day and delete all readings that are older than 1 day
+excluding the readings "day", "time" and "weekday"
+
+=over
+
+=item * param hash: hash of DWD_OpenData device
+
+=back
+
+=cut
+
+sub PruneForecast {
+  my ($hash) = @_;
+  my $name = $hash->{NAME};
+
+  if (::AttrVal($name, 'forecastPruning', 0) != 1) {
+    return;
+  }
+
+  ::Log3 $name, 5, "$name: PruneForecast: START";
+
+  my @readingNames = (grep {/^fc/} keys %{$hash->{READINGS}});
+
+  # find youngest timestamp per day
+  my %youngestTimestamps;
+  foreach my $readingName (@readingNames) {
+    if (!($readingName =~ m/^fc\d*_(day|time|weekday)$/)) {
+      my @parts = $readingName =~ /^fc(\d+)_.*/;
+      if (scalar(@parts) == 1) {
+        my $relativeDay = $parts[0];
+        my $timestamp = ::time_str2num($hash->{READINGS}{TIME});
+        my $youngestTimestamp = $youngestTimestamps{$relativeDay};
+        if (!defined($youngestTimestamp) || $timestamp > $youngestTimestamp) {
+          $youngestTimestamps{$relativeDay} = $timestamp;
+        }
+      }
+    }
+  }
+
+  # delete readings that are too old
+  foreach my $readingName (@readingNames) {
+    if (!($readingName =~ m/^fc\d*_(day|time|weekday)$/)) {
+      my @parts = $readingName =~ /^fc(\d+)_.*/;
+      if (scalar(@parts) == 1) {
+        my $relativeDay = $parts[0];
+        my $timestamp = ::time_str2num($hash->{READINGS}{TIME});
+        my $youngestTimestamp = $youngestTimestamps{$relativeDay};
+        if (defined($youngestTimestamp) && $timestamp < ($youngestTimestamp - 86400)) {
+          ::Log3 $name, 3, "$name: PruneForecast WARNING: deleting reading $readingName because it is more than 1 day older than all other readings of the same day";
+          ::CommandDeleteReading(undef, "$name $readingName");
+        }
+      }
+    }
+  }
+
+  ::Log3 $name, 5, "$name: PruneForecast: END";
+}
+
 sub ProcessForecast;
 
 =head2 GetForecast($$)
@@ -1781,10 +1854,10 @@ sub ProcessForecast {
     my @parts = split(/ at |\n/, $@); # discard anything after " at " or newline
     if (@parts) {
       $errorMessage = $parts[0];
-      ::Log3 $name, 4, "$name: ProcessForecast error: $parts[0]";
+      ::Log3 $name, 4, "$name: ProcessForecast ERROR: $parts[0]";
     } else {
       $errorMessage = $@;
-      ::Log3 $name, 4, "$name: ProcessForecast error: $@";
+      ::Log3 $name, 4, "$name: ProcessForecast ERROR: $@";
     }
   } else {
     # forecast parsed successfully
@@ -1802,7 +1875,7 @@ sub ProcessForecast {
       }
     } else {
       $errorMessage = 'result file name not defined';
-      ::Log3 $name, 3, "$name: ProcessForecast error: temp file name not defined";
+      ::Log3 $name, 3, "$name: ProcessForecast ERROR: temp file name not defined";
     }
   }
 
@@ -1843,7 +1916,7 @@ sub GetForecastFinish {
       # error, skip further processing
     } elsif (!defined($hash->{".forecastFile"})) {
       $errorMessage = "internal temp file name missing";
-      ::Log3 $name, 3, "$name: GetForecastFinish error: $errorMessage";
+      ::Log3 $name, 3, "$name: GetForecastFinish ERROR: $errorMessage";
     } else {
       # deserialize forecast
       my $fh = $hash->{".forecastFileHandle"};
@@ -1883,7 +1956,7 @@ sub GetForecastFinish {
 
     ::Log3 $name, 5, "$name: GetForecastFinish END";
   } else {
-    ::Log 3, "GetForecastFinish error: device name missing";
+    ::Log 3, "GetForecastFinish ERROR: device name missing";
   }
 }
 
@@ -1906,7 +1979,7 @@ sub GetForecastAbort {
   delete $hash->{".forecastBlockingCall"};
   delete $hash->{forecastUpdating};
   $errorMessage = "downloading and processing weather forecast data failed ($errorMessage)";
-  ::Log3 $name, 3, "$name: GetForecastAbort error: $errorMessage";
+  ::Log3 $name, 3, "$name: GetForecastAbort ERROR: $errorMessage";
 
   ::readingsBeginUpdate($hash);
   ::readingsBulkUpdate($hash, 'state', "forecast error: $errorMessage");
@@ -1948,7 +2021,7 @@ sub UpdateForecast {
 
   ::readingsBeginUpdate($hash);
 
-  # preprocess existing time readings
+  # preprocess existing readings
   my $time = time();
   my ($tSec, $tMin, $tHour, $tMday, $tMon, $tYear, $tWday, $tYday, $tIsdst) = Localtime($hash, $time);
   my $today = Timelocal($hash, 0, 0, 0, $tMday, $tMon, $tYear);
@@ -2045,7 +2118,7 @@ sub UpdateForecast {
     }
   }
 
-  # delete existing time readings of all days that have not been written
+  # delete readings of all days that have not been updated
   if ($relativeDay >= 0 && $daysAvailable > $relativeDay + 1) {
     ::Log3 $name, 5, "$name: deleting days with index " . ($relativeDay + 1) . " to " . ($daysAvailable - 1);
     for (my $d=($relativeDay + 1); $d<$daysAvailable; $d++) {
@@ -2053,6 +2126,9 @@ sub UpdateForecast {
     }
   }
 
+  # delete readings with inconsistent timestamps
+  PruneForecast($hash);
+
   ::readingsBulkUpdate($hash, 'state', 'forecast updated');
   ::readingsEndUpdate($hash, 1);
 
@@ -2321,10 +2397,10 @@ sub ProcessAlerts {
     my @parts = split(/ at |\n/, $@); # discard anything after " at " or newline
     if (@parts) {
       $errorMessage = $parts[0];
-      ::Log3 $name, 4, "$name: ProcessAlerts error: $parts[0]";
+      ::Log3 $name, 4, "$name: ProcessAlerts ERROR: $parts[0]";
     } else {
       $errorMessage = $@;
-      ::Log3 $name, 4, "$name: ProcessAlerts error: $@";
+      ::Log3 $name, 4, "$name: ProcessAlerts ERROR: $@";
     }
   } else {
     # alerts parsed successfully
@@ -2343,7 +2419,7 @@ sub ProcessAlerts {
       }
     } else {
       $errorMessage = 'result file name not defined';
-      ::Log3 $name, 3, "$name: ProcessAlerts error: temp file name not defined";
+      ::Log3 $name, 3, "$name: ProcessAlerts ERROR: temp file name not defined";
     }
   }
 
@@ -2388,7 +2464,7 @@ sub GetAlertsFinish {
       # error, skip further processing
     } elsif (!defined($hash->{".alertsFile".$communeUnion})) {
       $errorMessage = "internal temp file name missing";
-      ::Log3 $name, 3, "$name: GetAlertsFinish error: $errorMessage";
+      ::Log3 $name, 3, "$name: GetAlertsFinish ERROR: $errorMessage";
     } else {
       # deserialize alerts
       my $fh = $hash->{".alertsFileHandle".$communeUnion};
@@ -2458,7 +2534,7 @@ sub GetAlertsFinish {
 
     ::Log3 $name, 5, "$name: GetAlertsFinish END";
   } else {
-    ::Log 3, "GetAlertsFinish error: device name missing";
+    ::Log 3, "GetAlertsFinish ERROR: device name missing";
   }
 }
 
@@ -2483,7 +2559,7 @@ sub GetAlertsAbort {
   delete $hash->{".alertsBlockingCall".$communeUnion};
   $alertsUpdating[$communeUnion] = undef;
   $errorMessage = "downloading and processing weather alerts data failed ($errorMessage)";
-  ::Log3 $name, 3, "$name: GetAlertsAbort error: $errorMessage";
+  ::Log3 $name, 3, "$name: GetAlertsAbort ERROR: $errorMessage";
   $alertsErrorMessage[$communeUnion] = $errorMessage;
 
   if ($warncellId >= 0) {
@@ -2639,7 +2715,7 @@ sub DWD_OpenData_Initialize {
   $hash->{GetFn}      = 'DWD_OpenData::Get';
 
   $hash->{AttrList} = 'disable:0,1 '
-                      .'forecastStation forecastDays forecastProperties forecastResolution:1,3,6 forecastWW2Text:0,1 '
+                      .'forecastStation forecastDays forecastProperties forecastResolution:1,3,6 forecastWW2Text:0,1 forecastPruning:0,1 '
                       .'alertArea alertLanguage:DE,EN alertExcludeEvents '
                       .'timezone '
                       .$readingFnAttributes;
@@ -2653,6 +2729,13 @@ sub DWD_OpenData_Initialize {
 #
 # CHANGES
 #
+# 03.12.2020 (version 1.16.1) jensb
+# bugfix: delete destination reading if source reading is undefined when rotationg forecast at daybreak
+# feature: new attribute forecastPruning to delete forecast readings that are more than 1 day older than the other readings of the same day
+#
+# 22.11.2020 (version 1.15.0) jensb
+# feature: keep reading timestamp when rotating forecast values at daybreak
+#
 # 17.06.2020 (version 1.14.6) jensb
 # bugfix: $warncellId uninitialized when shutdown before first forecast fetch
 # coding: prototypes removed
@@ -2871,7 +2954,7 @@ sub DWD_OpenData_Initialize {
           Note: When value is changed all existing forecast readings will be deleted.
       </li><br>
       <li>forecastProperties [&lt;p1&gt;[,&lt;p2&gt;]...], default: Tx, Tn, Tg, TTT, DD, FX1, Neff, RR6c, RRhc, Rh00, ww<br>
-          A list of the properties available can be found <a href="https://opendata.dwd.de/weather/lib/MetElementDefinition.xml">here</a>.<br>
+          See the <a href="https://opendata.dwd.de/weather/lib/MetElementDefinition.xml">DWD forecast property defintions</a> for more details.<br>
           Notes:<br>
           - Not all properties are available for all stations and for all hours.<br>
           - If you remove a property from the list then already existing readings must be deleted manually in continuous mode.<br>
@@ -2879,6 +2962,12 @@ sub DWD_OpenData_Initialize {
       <li>forecastWW2Text {0|1}, default: 0<br>
           Create additional wwd readings containing the weather code as a descriptive text in German language.
       </li><br>
+      <li>forecastPruning {0|1}, default: 0<br>
+          Search for and delete forecast readings that are more then one day older then other forecast readings of the same day. Pruning will be performed after a successful forecast update.<br>
+          Notes:<br>
+          - Intended to maintain data consistency e.g. when a forecast station changes the reporting hour of a forecast property.<br>
+          - Requires noticable extra computing resources and may cause side effects if your FHEM configuration depends on a reading that is deleted.<br>
+      </li><br>
   </ul>
 
   <b>alert</b> related: