Add networking block

## [1.7.3](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/compare/v1.7.2...v1.7.3) (2021-04-12)

### Bug Fixes

* Empty commit to get recent changes into build (last commit should've been labeled this) ([e513752](e513752c46))

.gitlab-ci.yml

@@ -1,36 +1,112 @@
-# This file is a template, and might need editing before it works on your project.
+# You can override the included template(s) by including variable overrides
-# Official docker image.
+# See https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
-image: docker:latest
+# Note that environment variables can be set in several places
-
+# See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables
-services:
-  - docker:18.09-dind
 
 stages:
-  - build
+- test
+- build
+- push_local
+- release
+- push_dockerhub
 
-variables:
+.docker_base:
-  DOCKER_HOST: tcp://localhost:2375
+  image: docker:18.09.7-dind
-  DOCKER_DRIVER: overlay2
+  services:
-  #IMAGE_TAG: erichardson/py-eagle-mqtt:$CI_COMMIT_TAG
+    - docker:18.09.7-dind
-  IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
+  variables:
-
+    DOCKER_DRIVER: overlay
-before_script:
+    DOCKER_HOST: tcp://localhost:2375/
-  - docker version
+    IMAGE_TAG: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
-  - docker info
+    FF_GITLAB_REGISTRY_HELPER_IMAGE: 1
-  - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  before_script:
+    - docker version
+    - docker info
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
 
 build:
-  only:
+  extends:
-    - tags
+    - .docker_base
   stage: build
   script:
-    - docker build -t $IMAGE_TAG -t $CI_COMMIT_TAG ./Docker/
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
-    - docker push $IMAGE_TAG $CI_COMMIT_TAG
+  - >
+    docker build
+    --pull
+    --cache-from $CI_REGISTRY_IMAGE:latest
+    --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+    --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+    --build-arg VCS_REF=$CI_COMMIT_SHORT_SHA
+    ./Docker/
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
 
-#build:
+push local:
-#  stage: build
+  extends:
-#  script:
+    - .docker_base
-#    - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" .
+  stage: push_local
-#    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
+  variables:
-#  except:
+    GIT_STRATEGY: none
-#    - master
+  only:
+  - master
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:latest
+
+push dockerhub:
+  extends:
+    - .docker_base
+  stage: push_dockerhub
+  variables:
+    GIT_STRATEGY: none
+    CI_DOCKERHUB_IMAGE: index.docker.io/evanrich/py-eagle-mqtt
+    CI_DOCKERHUB_REGISTRY: docker.io
+  only:
+  - tags
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_DOCKERHUB_IMAGE:$CI_COMMIT_REF_NAME 
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_DOCKERHUB_IMAGE:latest
+  - docker login -u "$CI_DOCKERHUB_USER" -p "$CI_DOCKERHUB_PASSWORD" $CI_DOCKERHUB_REGISTRY
+  - docker push $CI_DOCKERHUB_IMAGE
+
+release:
+  image: node:13
+  stage: release
+  only:
+    refs:
+    - master
+    - alpha
+    # This matches maintenance branches
+    - /^(([0-9]+)\.)?([0-9]+)\.x/
+    # This matches pre-releases
+    - /^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/ 
+  script:
+    - touch CHANGELOG.md
+    - npm install @semantic-release/gitlab @semantic-release/changelog @semantic-release/git
+    - npx semantic-release
+  artifacts:
+    paths:
+    - CHANGELOG.md
+
+sast:
+  stage: test
+include:
+- template: Security/SAST.gitlab-ci.yml
+
+sonarqube-check:
+  image: 
+    name: sonarsource/sonar-scanner-cli:latest
+    entrypoint: [""]
+  variables:
+    SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache
+    GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
+  cache:
+    key: "${CI_JOB_NAME}"
+    paths:
+      - .sonar/cache
+  script: 
+    - sonar-scanner
+  allow_failure: true
+  only:
+    - master # or the name of your main branch

.gitlab/sast-ruleset.toml

@@ -0,0 +1,6 @@
+[bandit]
+  [[bandit.ruleset]]
+    disable = true
+    [bandit.ruleset.identifier]
+      type = "bandit_test_id"
+      value = "B101"

.releaserc.yml

@@ -0,0 +1,13 @@
+plugins:
+  - "@semantic-release/commit-analyzer"
+  - "@semantic-release/release-notes-generator"
+  - - "@semantic-release/changelog"
+    - changelogFile: CHANGELOG.md
+  - "@semantic-release/gitlab"
+  - - "@semantic-release/git"
+    - assets: CHANGELOG.md
+branches:
+  - "master"
+  - "+([0-9])?(.{+([0-9]),x}).x"
+  - name: "alpha"
+    prerelease: "alpha"

CHANGELOG.md

@@ -0,0 +1,75 @@
+## [1.7.3](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/compare/v1.7.2...v1.7.3) (2021-04-12)
+
+
+### Bug Fixes
+
+* Empty commit to get recent changes into build (last commit should've been labeled this) ([e513752](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/commit/e513752c46cc0b3beb93530f6549661d67196905))
+
+## [1.7.2](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/compare/v1.7.1...v1.7.2) (2021-04-08)
+
+
+### Bug Fixes
+
+* Cleanup CHANGELOG.md since using new semantic-release format ([a72ac7a](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/commit/a72ac7a1df91cb2231e38e8bbceb5ccf9c93328a))
+* fix asset info for git commit ([00c95c4](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/commit/00c95c42ea98921a1b2da9d2069d41e8e32f34f5))
+
+## [1.7.1](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/compare/v1.7.0...v1.7.1) (2021-04-08)
+
+
+### Bug Fixes
+
+* refactor Dockerfile to improve build ([40ac8c5](https://gitlab.evanrichardsonphotography.com/erichardson/py-eagle-mqtt/commit/40ac8c5ada9b993f53b72dcc14b2d9b7cd183e2f))
+
+## [1.6.10] - 2021-04-06
+
+### Fixed
+
+- Fix docker tagging issue
+
+## [1.6.9] - 2021-04-06
+
+### Fixed
+
+- Remove release-cli section of CI file
+- Add updated README.md
+
+## [1.6.8] - 2021-04-06
+
+### Changed
+
+- Fixed slight issue with tagging
+
+## [1.6.7] - 2021-04-06
+
+### Changed
+
+- Updated base image to Python 3.9.4-alpine3.13
+- Moved changes to this file
+
+### Removed
+
+- Changelog from README.md
+
+## [1.6.6] - 2021-04-03
+
+### Added
+
+- Added CI/CD pipeline to project
+
+### Changed
+
+- Moved project to my own personal Gitlab server for CI/CD pipelines as well as to run security scans on the container image and code
+
+- Updated base image to Python 3.9.3-alpine3.12
+
+## [Unversioned history]
+
+- 2019-07-24: Rebase to python 3.7.4-alpine3.10, update bottle to 0.12.17
+
+- 2019-04-12: Rebase to python 3.7.3-alpine3.9
+
+- 2019-03-08: Rebase to python 3.7.2-alpine3.9, update bottle and astral versions to latest
+
+- 2019-01-03: Rebase to python 3.7.2-alpine3.8, changed logging level to Info (should output to /var/log/tHome/eagle.log)
+
+- 2018-09-10: Ported to Python3, Added pricing info.  Merged into master branch

Docker/Dockerfile

@@ -1,17 +1,26 @@
-FROM python:3.7.4-alpine3.10
+FROM python:3.9.4-alpine3.13
-
 LABEL maintainer="Evan Richardson (evanrich81[at]gmail.com)"
-LABEL version="1.5.1"
+
+ARG BUILD_DATE
+ARG VCS_REF
+
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.build-date=$BUILD_DATE
+LABEL org.label-schema.name="evanrich/py-eagle-mqtt"
+LABEL org.label-schema.description="Python Rainforest Eagle to MQTT Application"
+LABEL org.label-schema.vcs-url="https://github.com/evanrich/py-eagle-mqtt"
+LABEL org.label-schema.vcs-ref=$VCS_REF
+LABEL org.label-schema.vendor="Evan Richardson"
+LABEL org.label-schema.docker.cmd="docker run --name=py-eagle-mqtt -e MQTT_BROKER_IP=<IP> -e MQTT_BROKER_PORT=1883 -p 22042:22042 -d evanrich/py-eagle-mqtt"
 
 WORKDIR /app
-COPY . /app
+COPY requirements.txt /app
 COPY ./src/conf/logrotate/tHome /etc/logrotate.d/
 ENV PYTHONPATH=/app/src/python
 RUN apk --update add --no-cache logrotate \
 && pip3 install --no-cache-dir  --upgrade pip \
 && pip3 install --no-cache-dir -r requirements.txt
-#RUN rm -rf /var/cache/apk
+COPY . /app
-
 CMD ["/app/src/bin/tHome-eagle.py", "-c", "/app/src/conf"]
 
 EXPOSE 22042

Docker/requirements.txt

@@ -1,4 +1,3 @@
-astral==1.10.1
+paho-mqtt==1.5.1
-paho-mqtt==1.4.0
+bottle==0.12.19
-bottle==0.12.17
+defusedxml==0.7.1
-

Docker/src/bin/tHome-eagle.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 
-#===========================================================================
+# ===========================================================================
 #
 # Eagle posting server 
 #
-#===========================================================================
+# ===========================================================================
 
 __doc__ = """
 Starts a small web server.  The Rain Forest Eagle is configured with
@@ -22,100 +22,116 @@ import json
 import bottle as B
 import tHome as T
 
-#===========================================================================
-def meter( client, data, cfg ):
-   msg = {
-      "time" : data.TimeUnix,
-      "consumed" : data.Consumed, # kWh
-      "produced" : data.Produced, # kWh
-      }
 
-   return ( cfg.mqttEnergy, msg )
+# ===========================================================================
+def meter(client, data, cfg):
+    msg = {
+        "time": data.TimeUnix,
+        "consumed": data.Consumed,  # kWh
+        "produced": data.Produced,  # kWh
+    }
 
-#===========================================================================
+    return cfg.mqttEnergy, msg
-def instant( client, data, cfg ):
-   msg = {
-      "time" : data.TimeUnix,
-      "power" : data.Power * 1000, # W
-      }
 
-   return ( cfg.mqttPower, msg )
-   
-#===========================================================================
-def price( client, data, cfg ):
-   msg = {
-      "time" : data.TimeUnix,
-      "price" : data.Price,
-      "tier" : data.Tier,
-      }
-   return ( cfg.mqttPrice, msg )
 
-#===========================================================================
+# ===========================================================================
+def instant(client, data, cfg):
+    msg = {
+        "time": data.TimeUnix,
+        "power": data.Power * 1000,  # W
+    }
+
+    return cfg.mqttPower, msg
+
+
+# ===========================================================================
+def price(client, data, cfg):
+    msg = {
+        "time": data.TimeUnix,
+        "price": data.Price,
+        "tier": data.Tier,
+    }
+    return cfg.mqttPrice, msg
+
+
+# ===========================================================================
+def network(client, data, cfg):
+    msg = {
+        "status": data.Status,
+        "description": data.Description,
+        "linkstrength": data.LinkStrength
+    }
+    return cfg.mqttNetwork, msg
+
+
+# ===========================================================================
 handlers = {
-   #"BlockPriceDetail" : 
+    # "BlockPriceDetail" :
-   "CurrentSummation" : meter,
+    "CurrentSummation": meter,
-   #"DeviceInfo" : 
+    # "DeviceInfo" :
-   #"FastPollStatus" : 
+    # "FastPollStatus" :
-   "InstantaneousDemand" : instant,
+    "InstantaneousDemand": instant,
-   #"MessageCluster" : 
+    # "MessageCluster" :
-   #"MeterInfo" : 
+    # "MeterInfo" :
-   #"NetworkInfo" : 
+    "NetworkInfo": network,
-   "PriceCluster" : price,
+    "PriceCluster": price,
-   #"Reading" : 
+    # "Reading" :
-   #"ScheduleInfo" :
+    # "ScheduleInfo" :
-   #"TimeCluster" : 
+    # "TimeCluster" :
-   }
+}
 
-#===========================================================================
 
-@B.post( '/' )
+# ===========================================================================
+
+@B.post('/')
 def root_post():
-   data = B.request.body.read( B.request.content_length )
+    data = B.request.body.read(B.request.content_length)
-   try:
+    try:
-      obj = T.eagle.parse( data )
+        obj = T.eagle.parse(data)
-   except:
+    except:
-      log.exception( "Error parsing Eagle posted data" )
+        log.exception("Error parsing Eagle posted data")
-      return "ERROR"
+        return "ERROR"
 
-   log.info( "Read packet: %s" % obj.name )
+    log.info("Read packet: %s" % obj.name)
-   
-   func = handlers.get( obj.name, None )
-   if func:
-      topic, msg = func( client, obj, cfg )
-      if msg:
-         log.info( "Publish: %s: %s" % ( topic, msg ) )
-         
-         payload = json.dumps( msg )
-         client.publish( topic, payload )
 
-   return "ok"
+    func = handlers.get(obj.name, None)
+    if func:
+        topic, msg = func(client, obj, cfg)
+        if msg:
+            log.info("Publish: %s: %s" % (topic, msg))
 
-#===========================================================================
+            payload = json.dumps(msg)
+            client.publish(topic, payload)
+
+    return "ok"
+
+
+# ===========================================================================
 #
 # Main applications script
 #
-#===========================================================================
+# ===========================================================================
 
-p = argparse.ArgumentParser( prog=sys.argv[0],
+p = argparse.ArgumentParser(prog=sys.argv[0],
-                             description="T-Home Eagle Server" )
+                            description="T-Home Eagle Server")
-p.add_argument( "-c", "--configDir", metavar="configDir",
+p.add_argument("-c", "--configDir", metavar="configDir",
-                default="/etc/tHome",
+               default="/etc/tHome",
-                help="Configuration file directory." )
+               help="Configuration file directory.")
-p.add_argument( "-l", "--log", metavar="logFile",
+p.add_argument("-l", "--log", metavar="logFile",
-                default=None, help="Logging file to use.  Input 'stdout' "
+               default=None, help="Logging file to use.  Input 'stdout' "
-                "to log to the screen." )
+                                  "to log to the screen.")
-c = p.parse_args( sys.argv[1:] )
+c = p.parse_args(sys.argv[1:])
 
 # Parse the eagle config file.
-cfg = T.eagle.config.parse( c.configDir )
+cfg = T.eagle.config.parse(c.configDir)
-log = T.eagle.config.log( cfg, c.log )
+log = T.eagle.config.log(cfg, c.log)
 
 # Create the MQTT client and connect it to the broker.
-client = T.broker.connect( c.configDir, log )
+client = T.broker.connect(c.configDir, log)
 
 # Start the MQTT as a background thread. This way we can run the web
 # server as the main thread here.
 client.loop_start()
 
-log.info( "Starting web server at port %d" % cfg.httpPort )
+log.info("Starting web server at port %d" % cfg.httpPort)
-B.run( host='0.0.0.0', port=cfg.httpPort, quiet=True )
+B.run(host='0.0.0.0', port=cfg.httpPort, quiet=True)

Docker/src/conf/broker.py

@@ -18,7 +18,7 @@ keepAlive = os.getenv('KEEPALIVE', 60)
 #
 #===========================================================================
 user = os.getenv('MQTT_USER', None)
-password = os.getenv('MQTT_PASS', None)
+password = os.getenv('MQTT_PASSWORD', None)
 
 #===========================================================================
 #
@@ -31,5 +31,3 @@ ca_certs = [
 
 certFile = None
 keyFile = None
-
-

Docker/src/conf/eagle.py

@@ -23,6 +23,9 @@ mqttPrice = 'power/elec/Home/price'
 #Current rate label (returns rate label from meter)
 mqttRateLabel = 'power/elec/Home/ratelabel'
 
+#Network Info Topic (returns status, description and link strength from meter)
+mqttNetwork = 'power/elec/Home/network'
+
 #===========================================================================
 #
 # Logging configuration.  Env variables are allowed in the file name.

Docker/src/python/tHome/eagle/get.py

@@ -6,7 +6,7 @@ from . import messages as msg
 #from .InstantDemand import InstantDemand
 #from .Reading import Reading
 #from .Total import Total
-import xml.etree.ElementTree as ET
+import defusedxml.ElementTree as ET
 import socket
 
 #==========================================================================

Docker/src/python/tHome/eagle/parse.py

@@ -3,7 +3,7 @@
 # Parse XML messages into an object.
 #
 #===========================================================================
-import xml.etree.ElementTree as ET
+import defusedxml.ElementTree as ET
 from . import messages
 
 #==========================================================================

README.md

@@ -1,19 +1,20 @@
 # py-eagle-mqtt
+
 Python3 based Docker for Eagle to MQTT reader
 
-I have only ported this into a dockerfile, All Original code (with one minor modification for XML root tag) is credit to [Ted Drain - TD22057](https://github.com/TD22057/T-Home).
+I have only ported this into a dockerfile, as well as made some changes to code for security or other purposes. All Original code is credit to [Ted Drain - TD22057](https://github.com/TD22057/T-Home).
+
+This project utilizes the following tools:
+
+[![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release)
+[![Libraries.io dependency status for GitHub repo](https://img.shields.io/librariesio/github/evanrich/py-eagle-mqtt)]()
+[![Gitlab pipeline status (self-hosted)](https://img.shields.io/gitlab/pipeline/erichardson/py-eagle-mqtt/master?gitlab_url=https%3A%2F%2Fgitlab.evanrichardsonphotography.com)]()
+[![Docker Pulls](https://img.shields.io/docker/pulls/evanrich/py-eagle-mqtt)]()
+[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/evanrich/py-eagle-mqtt)]()
 
 ## UPDATES:
-2019-07-24: Rebase to python 3.7.4-alpine3.10, update bottle to 0.12.17
-
-2019-04-12: Rebase to python 3.7.3-alpine3.9
-
-2019-03-08: Rebase to python 3.7.2-alpine3.9, update bottle and astral versions to latest
-
-2019-01-03: Rebase to python 3.7.2-alpine3.8, changed logging level to Info (should output to /var/log/tHome/eagle.log)
-
-2018-09-10: Ported to Python3, Added pricing info.  Merged into master branch
 
+2020-04-06: Moved Updates to a CHANGELOG.md file to clean this up.
 
 ## Usage
 

k8s/deployment.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: py-eagle-mqtt

sonar-project.properties

@@ -0,0 +1,3 @@
+sonar.projectKey=erichardson_py-eagle-mqtt_AXkgI9tRLcemhRz3NCjo
+sonar.qualitygate.wait=true
+sonar.projectBaseDir=Docker/src