Merge pull request #613 from geeksville/pr-moveproto

Move protobuf python glue from meshtastic to meshtastic.protobuf

Author: ianmcorvidae

.coveragerc

@@ -1,5 +1,5 @@
 [run]
-omit = meshtastic/*_pb2.py,meshtastic/tests/*.py,meshtastic/test.py
+omit = meshtastic/protobuf/*_pb2.py,meshtastic/tests/*.py,meshtastic/test.py
 
 [report]
 exclude_lines =

.github/workflows/update_protobufs.yml

@@ -37,5 +37,5 @@ jobs:
           git config --global user.email 'bot@noreply.github.com'
           git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
           git add protobufs
-          git add meshtastic
+          git add meshtastic/protobuf
           git commit -m "Update protobuf submodule" && git push || echo "No changes to commit"

bin/regen-protobufs.sh

@@ -1,23 +1,53 @@
 #!/bin/bash
 
+set -e
+
 #Uncomment to run hack
 #gsed -i 's/import "\//import ".\//g' ./protobufs/meshtastic/*
 #gsed -i 's/package meshtastic;//g' ./protobufs/meshtastic/*
 
 # protoc looks for mypy plugin in the python path
 source $(poetry env info --path)/bin/activate
 
-./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./ --mypy_out ./ ./protobufs/meshtastic/*.proto
-./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./meshtastic/ --mypy_out ./meshtastic/ ./protobufs/nanopb.proto
+# Put our temp files in the poetry build directory
+TMPDIR=./build/meshtastic/protofixup
+echo "Fixing up protobuf paths in ${TMPDIR} temp directory"
+
+
+# Ensure a clean build
+rm -r "${TMPDIR}"
 
-# workaround for import bug in protoc https://github.com/protocolbuffers/protobuf/issues/1491#issuecomment-690618628
+INDIR=${TMPDIR}/in/meshtastic/protobuf
+OUTDIR=${TMPDIR}/out
+PYIDIR=${TMPDIR}/out
+mkdir -p "${OUTDIR}" "${INDIR}" "${PYIDIR}"
+cp ./protobufs/meshtastic/*.proto "${INDIR}"
 
+# OS-X sed is apparently a little different and expects an arg for -i
 if [[ $OSTYPE == 'darwin'* ]]; then
-	sed -i '' -E 's/^(import.*_pb2)/from . \1/' meshtastic/*.py
-	# automate the current workaround (may be related to Meshtastic-protobufs issue #27 https://github.com/meshtastic/protobufs/issues/27)
-	sed -i '' -E "s/^None = 0/globals()['None'] = 0/" meshtastic/mesh_pb2.py
+	SEDCMD="sed -i '' -E"
 else
-	sed -i -e 's/^import.*_pb2/from . \0/' meshtastic/*.py
-	# automate the current workaround (may be related to Meshtastic-protobufs issue #27 https://github.com/meshtastic/protobufs/issues/27)
-	sed -i -e "s/^None = 0/globals()['None'] = 0/" meshtastic/mesh_pb2.py
+	SEDCMD="sed -i -E"
 fi
+
+
+# change the package names to meshtastic.protobuf
+$SEDCMD 's/^package meshtastic;/package meshtastic.protobuf;/' "${INDIR}/"*.proto
+# fix the imports to match
+$SEDCMD 's/^import "meshtastic\//import "meshtastic\/protobuf\//' "${INDIR}/"*.proto
+
+# Generate the python files
+./nanopb-0.4.8/generator-bin/protoc -I=$TMPDIR/in --python_out "${OUTDIR}" "--mypy_out=${PYIDIR}" $INDIR/*.proto
+
+# Change "from meshtastic.protobuf import" to "from . import"
+$SEDCMD 's/^from meshtastic.protobuf import/from . import/' "${OUTDIR}"/meshtastic/protobuf/*pb2*.py[i]
+
+# Create a __init__.py in the out directory
+touch "${OUTDIR}/meshtastic/protobuf/__init__.py"
+
+# Copy to the source controlled tree
+mkdir -p meshtastic/protobuf
+rm -rf meshtastic/protobuf/*pb2*.py
+cp "${OUTDIR}/meshtastic/protobuf"/* meshtastic/protobuf
+
+exit 0

meshtastic/__init__.py

@@ -81,7 +81,10 @@ def onConnection(interface, topic=pub.AUTO_TOPIC): # called when we (re)connect
 from pubsub import pub # type: ignore[import-untyped]
 from tabulate import tabulate
 
-from meshtastic import (
+from meshtastic.node import Node
+from meshtastic.util import DeferredExecution, Timeout, catchAndIgnore, fixme, stripnl
+
+from .protobuf import (
     admin_pb2,
     apponly_pb2,
     channel_pb2,
@@ -93,10 +96,10 @@ def onConnection(interface, topic=pub.AUTO_TOPIC): # called when we (re)connect
     remote_hardware_pb2,
     storeforward_pb2,
     telemetry_pb2,
+)
+from . import (
     util,
 )
-from meshtastic.node import Node
-from meshtastic.util import DeferredExecution, Timeout, catchAndIgnore, fixme, stripnl
 
 # Note: To follow PEP224, comments should be after the module variable.
 

meshtastic/__main__.py

@@ -17,7 +17,8 @@
 import meshtastic.test
 import meshtastic.util
 from meshtastic import mt_config
-from meshtastic import channel_pb2, config_pb2, portnums_pb2, remote_hardware, BROADCAST_ADDR
+from meshtastic.protobuf import channel_pb2, config_pb2, portnums_pb2
+from meshtastic import remote_hardware, BROADCAST_ADDR
 from meshtastic.version import get_active_version
 from meshtastic.ble_interface import BLEInterface
 from meshtastic.mesh_interface import MeshInterface