From 01b4778f51e48d2fcc766a682ee95cd828e3b879 Mon Sep 17 00:00:00 2001 From: Simon <misc@poweron.dk> Date: Wed, 1 Mar 2023 01:31:39 +0100 Subject: [PATCH] Minor cleanup/adjustments & change of name --- .gitignore | 1 + README.md | 29 ++++++++++--------- docker-compose.yml | 16 +++++----- docker-compose_nginxphp.yml | 12 ++++---- src/ct_grafana/template_dashboard.json | 2 +- src/ct_nginx/nginx.conf | 2 +- src/php_app/app/{Elo => powerflux}/Common.php | 11 ++----- .../app/{Elo => powerflux}/Configuration.php | 21 +++++--------- .../app/{Elo => powerflux}/ErrorOutput.php | 24 ++++----------- .../app/{Elo => powerflux}/Files/File.php | 6 ++-- .../app/{Elo => powerflux}/Files/FileApp.php | 8 ++--- .../{Elo => powerflux}/Files/FileSettings.php | 8 ++--- .../app/{Elo => powerflux}/Influxdb.php | 8 ++--- src/php_app/app/{Elo => powerflux}/Parser.php | 7 ++--- src/php_app/app/{Elo => powerflux}/Prices.php | 2 +- src/php_app/composer.json | 2 +- src/php_app/public/get_ancient.php | 2 +- src/php_app/public/update.php | 2 +- 18 files changed, 68 insertions(+), 95 deletions(-) rename src/php_app/app/{Elo => powerflux}/Common.php (98%) rename src/php_app/app/{Elo => powerflux}/Configuration.php (96%) rename src/php_app/app/{Elo => powerflux}/ErrorOutput.php (91%) rename src/php_app/app/{Elo => powerflux}/Files/File.php (90%) rename src/php_app/app/{Elo => powerflux}/Files/FileApp.php (90%) rename src/php_app/app/{Elo => powerflux}/Files/FileSettings.php (90%) rename src/php_app/app/{Elo => powerflux}/Influxdb.php (97%) rename src/php_app/app/{Elo => powerflux}/Parser.php (98%) rename src/php_app/app/{Elo => powerflux}/Prices.php (99%) diff --git a/.gitignore b/.gitignore index 154af80..e3410ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.vscode /eloparser_docker.code-workspace eloparser_public.code-workspace +powerflux.code-workspace /composer.phar /CHANGELOG.md /src/php_app/app.json diff --git a/README.md b/README.md index fcf6b85..c17871b 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# eloparser +# PowerFlux ## Fetches data and writes to a InfluxDB database - Meter readings from Eloverblik (Power consumption per hour) - Spot prices from Energidataservice ->**Version: v0.54** +>**Version: v0.55** --- ### JSON Output from update.php @@ -28,7 +28,7 @@ Various info will be available, for use in Annotations in Grafana or for whateve ### [Install using Docker Compose (Recommended)](#install-using-docker-compose-recommended) -#### For the full stack - PHP app, webserver (nginx), database (InfluxDB) and data visualization (Grafana): +#### For the full stack - PHP app, webserver (nginx), database (InfluxDB) and data visualization (Grafana) 1. Insert your refresh token as a environment variable in docker-compose.yml 2. Run Docker Compose on docker-compose.yml @@ -39,9 +39,9 @@ Various info will be available, for use in Annotations in Grafana or for whateve - JSON output from PHP app is available on <http://localhost:8181> -##### Already got InfluxDB and Grafana? +#### Already got InfluxDB and Grafana? -##### docker-compose_nginxphp.yml provides a stack with only nginx and the PHP app and has environment variables set up ready for use in Portainer. +- Use docker-compose_nginxphp.yml - It provides a stack with only nginx and the PHP app and has environment variables set up ready for use in Portainer. ### [Manual installation](#manual-installation) @@ -49,7 +49,7 @@ Various info will be available, for use in Annotations in Grafana or for whateve 2. Run get_ancient.php to get "ancient" data (all years before current) - preferably in CLI to avoid problems with timeouts \ Please note that you only have to run get_ancient.php 1 time. Unless you have a time machine its unlikely that your power usage back in etc 2015 changes :D -3. Run update.php *(etc every 6 hours in a cronjob for continuously data retrieval)* +3. Run update.php *(etc every 1 hour in a cronjob to ensure continuously data retrieval)* --- @@ -58,7 +58,7 @@ Please note that you only have to run get_ancient.php 1 time. Unless you have a ### [Example of correct settings.json](#example-of-correct-settingsjson) -#### Not relevant if you use Docker. +#### Not relevant if you use Docker ```json { @@ -104,16 +104,19 @@ Please note that you only have to run get_ancient.php 1 time. Unless you have a ## Notes -[^first]: update.php will attempt to fetch new data from Eloverblik every 8 hours (*updateinterval* in Configuration class), for the latest entry in the InfluxDB database to the current date. Please note that Eloverblik will always provide data with 2~ days delay. \ -You can call this code every few seconds if you like to, it will only do calls to Eloverblik after 8 hours has passed since last successful request. \ +[^first]: update.php will attempt to fetch new data from Eloverblik every 6 hours (*updateinterval* in Configuration class), for the latest entry in the InfluxDB database to the current date. Please note that Eloverblik will always provide data with 2~ days delay. \ +You can call this code every few seconds if you like to, it will only do calls to Eloverblik after 6 hours has passed since last successful request. \ Docker version runs update.php every minute (a bit overkill yes). -- The code has been tested on Linux and Windows, on Windows you need to escape the paths specified in update.php, with \ or just use / instead, example: ```D:\\aFolder\anotherOne\\test1\\``` and ```D:/aFolder/anotherOne/test1/```should work. +- The PHP code and the Docker stack has been tested on Linux(Debian & Photon OS) and Windows 10 (x86-64). +- On Windows you need to escape the paths specified in update.php, with \ or just use / instead, example: ```D:\\aFolder\anotherOne\\test1\\``` and ```D:/aFolder/anotherOne/test1/```should work. -## Grafana and InfluxDB (Suggestion for manual installation) +## Grafana and InfluxDB (Suggestions for manual installation) If you want to use InfluxQL with InfluxDB 2.0+ you may need to create a DBRP mapping. \ List buckets with: ```influx v1 dbrp list``` \ Create bucket mapping with: ```influx v1 dbrp create --bucket-id yourBucketID --db yourBucket --rp autogen``` \ -<https://docs.influxdata.com/influxdb/v2.4/query-data/influxql/dbrp/#create-dbrp-mappings> \ -This only applies to InfluxQL, for Flux queries you don't need to do anything. \ No newline at end of file +<https://docs.influxdata.com/influxdb/v2.5/query-data/influxql/dbrp/#create-dbrp-mappings> \ +This only applies to InfluxQL, for Flux queries you don't need to do anything. + + Grafana - JSON source plugin, etc for use of JSON output: Infinity plugin - <https://grafana.com/grafana/plugins/yesoreyeram-infinity-datasource/> \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d176994..3f31eb4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,9 +7,9 @@ services: dockerfile: ./ct_nginx/Dockerfile working_dir: /app volumes: - - vol_eloparser:/app + - vol_powerflux:/app networks: - - elostack + - powerfluxstack ports: - "8181:80" @@ -20,9 +20,9 @@ services: context: ./src dockerfile: ./ct_php-fpm/Dockerfile volumes: - - vol_eloparser:/app + - vol_powerflux:/app networks: - - elostack + - powerfluxstack environment: REFRESH_TOKEN: "yourRefreshTokenfromEloverblik" #For Portainer: ${REFRESH_TOKEN} INFLUXDB_URL: "http://influxdb:8086" #For Portainer: ${INFLUXDB_URL} @@ -39,7 +39,7 @@ services: - vol_influxdb_config:/etc/influxdb2 - vol_influxdb_data:/var/lib/influxdb2 networks: - - elostack + - powerfluxstack environment: INFLUXD_LOG_LEVEL: "info" INFLUXD_REPORTING_DISABLED: "true" @@ -64,7 +64,7 @@ services: volumes: - vol_grafana:/var/lib/grafana networks: - - elostack + - powerfluxstack environment: GF_ANALYTICS_REPORTING_ENABLED: "false" GF_AUTH_ANONYMOUS_ENABLED: "false" #You can switch this to true if you want to enable anonymous access (no login required). @@ -78,10 +78,10 @@ services: - influxdb volumes: - vol_eloparser: + vol_powerflux: vol_influxdb_config: vol_influxdb_data: vol_grafana: networks: - elostack: \ No newline at end of file + powerfluxstack: \ No newline at end of file diff --git a/docker-compose_nginxphp.yml b/docker-compose_nginxphp.yml index bad523c..3c15f9d 100644 --- a/docker-compose_nginxphp.yml +++ b/docker-compose_nginxphp.yml @@ -7,9 +7,9 @@ services: dockerfile: ./ct_nginx/Dockerfile working_dir: /app volumes: - - vol_eloparser:/app + - vol_powerflux:/app networks: - - elostack + - powerfluxstack ports: - "8181:80" @@ -20,9 +20,9 @@ services: context: ./src dockerfile: ./ct_php-fpm/Dockerfile volumes: - - vol_eloparser:/app + - vol_powerflux:/app networks: - - elostack + - powerfluxstack environment: REFRESH_TOKEN: ${REFRESH_TOKEN} # Portainer uses this to replace it with values from webinterface INFLUXDB_URL: ${INFLUXDB_URL} # If you don't want that see docker-compose.yml for a example of setting the values directly in here @@ -31,7 +31,7 @@ services: INFLUXDB_ORG: ${INFLUXDB_ORG} volumes: - vol_eloparser: + vol_powerflux: networks: - elostack: \ No newline at end of file + powerfluxstack: \ No newline at end of file diff --git a/src/ct_grafana/template_dashboard.json b/src/ct_grafana/template_dashboard.json index 8a7d324..8743753 100644 --- a/src/ct_grafana/template_dashboard.json +++ b/src/ct_grafana/template_dashboard.json @@ -422,7 +422,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Template Dashboard (Eloverblik)", + "title": "Template Dashboard (PowerFlux)", "uid": "lS1LLcv4k", "version": 1, "weekStart": "" diff --git a/src/ct_nginx/nginx.conf b/src/ct_nginx/nginx.conf index 75f07aa..3f41f02 100644 --- a/src/ct_nginx/nginx.conf +++ b/src/ct_nginx/nginx.conf @@ -1,7 +1,7 @@ server { listen 80 default; - client_max_body_size 128M; + client_max_body_size 16M; access_log /var/log/nginx/app_access.log; diff --git a/src/php_app/app/Elo/Common.php b/src/php_app/app/powerflux/Common.php similarity index 98% rename from src/php_app/app/Elo/Common.php rename to src/php_app/app/powerflux/Common.php index 137f498..46b0bc3 100644 --- a/src/php_app/app/Elo/Common.php +++ b/src/php_app/app/powerflux/Common.php @@ -1,6 +1,6 @@ <?php -namespace Elo; +namespace PowerFlux; use DateInterval; use DateTimeImmutable; @@ -102,7 +102,7 @@ class Common array_push($r, array("start"=>$yearstart->format($format), "stop"=> $yearend->format($format) ) ); } - //tmp fix + //tmp fix - Todo: Rework above if ( Common::getTimeDate(str: end($r)["start"], format: "Y") !== Common::getTimeDate(format: "Y") ) { $thisyear = Common::getDateTime("now"); $thisyear = $thisyear->modify("first day of January"); @@ -111,8 +111,6 @@ $end = $end->modify("+1 day"); //Tomorrow array_push($r, array("start"=>$thisyear->format($format), "stop"=> $end->format($format) ) ); } - - return $r ?? null; } @@ -238,11 +236,6 @@ array_push($r, array("start"=>$thisyear->format($format), "stop"=> $end->format( return null; } - public static function getDeviations(?int $deviation, $ts1, $ts2): ?int - { - return null; - } - /** * Get time difference in seconds between two Unix timestamps. //Todo, remove */ diff --git a/src/php_app/app/Elo/Configuration.php b/src/php_app/app/powerflux/Configuration.php similarity index 96% rename from src/php_app/app/Elo/Configuration.php rename to src/php_app/app/powerflux/Configuration.php index 31225fd..1f09bf5 100644 --- a/src/php_app/app/Elo/Configuration.php +++ b/src/php_app/app/powerflux/Configuration.php @@ -1,15 +1,15 @@ <?php -namespace Elo; +namespace PowerFlux; -use Elo\Files\File; -use Elo\Files\FileApp; -use Elo\Files\FileSettings; +use PowerFlux\Files\File; +use PowerFlux\Files\FileApp; +use PowerFlux\Files\FileSettings; class Configuration { private static $apiurl = "https://api.eloverblik.dk/customerapi/"; private static $apiurl_prices = "https://api.energidataservice.dk/"; -private static $updateinterval = 480; //mins - Interval between attempts at fetching live data from Eloverblik (8 hours) +private static $updateinterval = 360; //mins - Interval between attempts at fetching live data from Eloverblik (6 hours) private static $path_settings = ""; private static $path_app = ""; private static $path_data_response = __DIR__ . "/../../"; @@ -49,13 +49,6 @@ public const defaultpath = __DIR__ . "/../../"; //Relative path return $secs ?? null; } -public static function hasMeterData(): bool { //Todo: remove - only checks if there is 'data' stored in data_raw.json - if ($d = File::getFile("dataresp")) { - return true; - } - return false; -} - public static function getRefreshToken() //Todo: Check if valid token { return self::getSettingsEntry("refresh_token"); @@ -356,7 +349,7 @@ public static function hasMeterData(): bool { //Todo: remove - only checks if th } return self::$toggledebug; } -#endregion + #endregion #region Files @@ -409,5 +402,5 @@ public static function hasMeterData(): bool { //Todo: remove - only checks if th return self::$_instance; } -} //end class +} diff --git a/src/php_app/app/Elo/ErrorOutput.php b/src/php_app/app/powerflux/ErrorOutput.php similarity index 91% rename from src/php_app/app/Elo/ErrorOutput.php rename to src/php_app/app/powerflux/ErrorOutput.php index 769b467..25cf0d1 100644 --- a/src/php_app/app/Elo/ErrorOutput.php +++ b/src/php_app/app/powerflux/ErrorOutput.php @@ -1,5 +1,5 @@ <?php -namespace Elo; +namespace PowerFlux; enum ERRORCODES: int @@ -170,9 +170,6 @@ class ErrorOutput $infostr = "Couldn't authenticate with refresh token, please check that the refresh token is in fact valid and active."; Configuration::setAppEntry("ts_api_readings_lastfail", Common::getTime()); ErrorOutput::ERROR($infostr); - if (!Configuration::hasMeterData()) { - exit($infostr); - } } else { $infostr = "Unhandled error occured in " . $func . " - 401 Unauthorized"; ErrorOutput::ERROR($infostr); @@ -210,15 +207,9 @@ class ErrorOutput break; case ERRORCODES::HTTP503: - if (Configuration::hasMeterData()) { //Todo: remove, Has stored data so log the incident and continue - $infostr = "DataHub is unavailable."; - ErrorOutput::INFO($infostr); - } else { - $infostr = "DataHub is unavailable, unable to continue - no stored data is available."; //Todo: Rework after influx version - Configuration::setAppEntry("ts_api_readings_lastfail", Common::getTime()); - ErrorOutput::WARNING($infostr); - exit($infostr); - } + $infostr = "DataHub is unavailable."; + Configuration::setAppEntry("ts_api_readings_lastfail", Common::getTime()); + ErrorOutput::WARNING($infostr); break; case ERRORCODES::APP_NOREFRESH: @@ -254,12 +245,7 @@ class ErrorOutput case ERRORCODES::APP_INVALID_RESPONSE: $infostr = "Received invalid or empty response from Eloverblik."; Configuration::setAppEntry("ts_api_readings_lastfail", Common::getTime()); - if (Configuration::hasMeterData()) { //Todo: remove, not relevant anymore - ErrorOutput::NOTICE($infostr); - } else { - ErrorOutput::NOTICE($infostr); - exit($infostr); - } + ErrorOutput::NOTICE($infostr); break; case ERRORCODES::APP_NOSETTINGS: diff --git a/src/php_app/app/Elo/Files/File.php b/src/php_app/app/powerflux/Files/File.php similarity index 90% rename from src/php_app/app/Elo/Files/File.php rename to src/php_app/app/powerflux/Files/File.php index fb630d7..f772d27 100644 --- a/src/php_app/app/Elo/Files/File.php +++ b/src/php_app/app/powerflux/Files/File.php @@ -1,8 +1,8 @@ <?php -namespace Elo\Files; +namespace PowerFlux\Files; -use Elo\Configuration; -use Elo\ErrorOutput; +use PowerFlux\Configuration; +use PowerFlux\ErrorOutput; class File { diff --git a/src/php_app/app/Elo/Files/FileApp.php b/src/php_app/app/powerflux/Files/FileApp.php similarity index 90% rename from src/php_app/app/Elo/Files/FileApp.php rename to src/php_app/app/powerflux/Files/FileApp.php index e1e3b5f..398916d 100644 --- a/src/php_app/app/Elo/Files/FileApp.php +++ b/src/php_app/app/powerflux/Files/FileApp.php @@ -1,9 +1,9 @@ <?php -namespace Elo\Files; +namespace PowerFlux\Files; -use Elo\Configuration; -use Elo\ERRORCODES; -use Elo\ErrorOutput; +use PowerFlux\Configuration; +use PowerFlux\ERRORCODES; +use PowerFlux\ErrorOutput; class FileApp extends File { diff --git a/src/php_app/app/Elo/Files/FileSettings.php b/src/php_app/app/powerflux/Files/FileSettings.php similarity index 90% rename from src/php_app/app/Elo/Files/FileSettings.php rename to src/php_app/app/powerflux/Files/FileSettings.php index 162c8c4..d38bac4 100644 --- a/src/php_app/app/Elo/Files/FileSettings.php +++ b/src/php_app/app/powerflux/Files/FileSettings.php @@ -1,9 +1,9 @@ <?php -namespace Elo\Files; +namespace PowerFlux\Files; -use Elo\Configuration; -use Elo\ERRORCODES; -use Elo\ErrorOutput; +use PowerFlux\Configuration; +use PowerFlux\ERRORCODES; +use PowerFlux\ErrorOutput; class FileSettings extends File { diff --git a/src/php_app/app/Elo/Influxdb.php b/src/php_app/app/powerflux/Influxdb.php similarity index 97% rename from src/php_app/app/Elo/Influxdb.php rename to src/php_app/app/powerflux/Influxdb.php index b45882c..0c45411 100644 --- a/src/php_app/app/Elo/Influxdb.php +++ b/src/php_app/app/powerflux/Influxdb.php @@ -1,5 +1,5 @@ <?php -namespace Elo; +namespace PowerFlux; use DateTimeImmutable; use DateTimeZone; @@ -33,7 +33,7 @@ class Influxdb { } } -public static function isValidURL($str): ?string { //Todo: Checks that this is somewhat valid +public static function isValidURL($str): ?string { //Todo: move to common //Todo: Checks that this is somewhat valid if (strlen($str) >= 10) { return $str; } @@ -105,7 +105,7 @@ public static function isValidBucket($str): string { } } } - return $r ?? null; //ErrorOutput::handleError(ERRORCODES::INFLUX_GETLATEST); + return $r ?? null; } public static function doAncientPeriods($periods, $meter) { @@ -128,8 +128,6 @@ public static function isValidBucket($str): string { } - - /** Writes meter readings to InfluxDB database. * */ diff --git a/src/php_app/app/Elo/Parser.php b/src/php_app/app/powerflux/Parser.php similarity index 98% rename from src/php_app/app/Elo/Parser.php rename to src/php_app/app/powerflux/Parser.php index c2d501f..5ddb483 100644 --- a/src/php_app/app/Elo/Parser.php +++ b/src/php_app/app/powerflux/Parser.php @@ -1,9 +1,9 @@ <?php -namespace Elo; +namespace PowerFlux; use DateTimeImmutable; use DateTimeZone; -use Elo\Files\File; +use PowerFlux\Files\File; use GuzzleHttp\Client; use GuzzleHttp\Psr7; use GuzzleHttp\Exception\ClientException; @@ -15,7 +15,6 @@ class Parser { public function __construct() { - // } @@ -265,7 +264,7 @@ class Parser { $meterid = Configuration::getSettingsEntry("meter_id") ?: ErrorOutput::handleError(ERRORCODES::APP_INVALID_METERID ?? ERRORCODES::UNKNOWN); $queryurl = $appendix . $datefrom . "/" . $dateto . "/" . $timeaggr; - //Todo: Check with regex if queryurl equals the correct format + //Todo: use common method; Todo: Check with regex if queryurl equals the correct format ErrorOutput::INFO("Date from " . $datefrom . " to " . $dateto . " (meter: " . $meterid. ")"); $querybody = json_encode( array("meteringPoints" => array("meteringPoint" => array( $meterid ) )), JSON_PRETTY_PRINT); diff --git a/src/php_app/app/Elo/Prices.php b/src/php_app/app/powerflux/Prices.php similarity index 99% rename from src/php_app/app/Elo/Prices.php rename to src/php_app/app/powerflux/Prices.php index 1ae524d..5dee8fc 100644 --- a/src/php_app/app/Elo/Prices.php +++ b/src/php_app/app/powerflux/Prices.php @@ -1,6 +1,6 @@ <?php -namespace Elo; +namespace PowerFlux; use GuzzleHttp\Exception\ClientException; use GuzzleHttp\Exception\ServerException; diff --git a/src/php_app/composer.json b/src/php_app/composer.json index aefef17..ff6c0bc 100644 --- a/src/php_app/composer.json +++ b/src/php_app/composer.json @@ -9,6 +9,6 @@ "influxdata/influxdb-client-php": "^2.9" }, "autoload": { - "psr-4": { "Elo\\" : "app/Elo" } + "psr-4": { "PowerFlux\\" : "app/powerflux" } } } diff --git a/src/php_app/public/get_ancient.php b/src/php_app/public/get_ancient.php index 78113a2..e650c7d 100644 --- a/src/php_app/public/get_ancient.php +++ b/src/php_app/public/get_ancient.php @@ -1,5 +1,5 @@ <?php -namespace Elo; +namespace PowerFlux; require_once(__DIR__ . '/../vendor/autoload.php'); header('content-type: application/json'); diff --git a/src/php_app/public/update.php b/src/php_app/public/update.php index e1c2397..28dda21 100644 --- a/src/php_app/public/update.php +++ b/src/php_app/public/update.php @@ -1,5 +1,5 @@ <?php -namespace Elo; +namespace PowerFlux; require_once(__DIR__ . '/../vendor/autoload.php'); header('content-type: application/json'); -- GitLab