Move protobufs to meshtastic.protobuf python namespace

Author: geeksville

.github/workflows/update_protobufs.yml

@@ -37,5 +37,5 @@ jobs:
           git config --global user.email 'bot@noreply.github.com'
           git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
           git add protobufs
-          git add meshtastic
+          git add meshtastic/protobuf
           git commit -m "Update protobuf submodule" && git push || echo "No changes to commit"

bin/regen-protobufs.sh

@@ -1,23 +1,53 @@
 #!/bin/bash
 
+set -e
+
 #Uncomment to run hack
 #gsed -i 's/import "\//import ".\//g' ./protobufs/meshtastic/*
 #gsed -i 's/package meshtastic;//g' ./protobufs/meshtastic/*
 
 # protoc looks for mypy plugin in the python path
 source $(poetry env info --path)/bin/activate
 
-./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./ --mypy_out ./ ./protobufs/meshtastic/*.proto
-./nanopb-0.4.8/generator-bin/protoc -I=protobufs --python_out ./meshtastic/ --mypy_out ./meshtastic/ ./protobufs/nanopb.proto
+# Put our temp files in the poetry build directory
+TMPDIR=./build/meshtastic/protofixup
+echo "Fixing up protobuf paths in ${TMPDIR} temp directory"
+
+
+# Ensure a clean build
+rm -r "${TMPDIR}"
 
-# workaround for import bug in protoc https://github.com/protocolbuffers/protobuf/issues/1491#issuecomment-690618628
+INDIR=${TMPDIR}/in/meshtastic/protobuf
+OUTDIR=${TMPDIR}/out
+PYIDIR=${TMPDIR}/out
+mkdir -p "${OUTDIR}" "${INDIR}" "${PYIDIR}"
+cp ./protobufs/meshtastic/*.proto "${INDIR}"
 
+# OS-X sed is apparently a little different and expects an arg for -i
 if [[ $OSTYPE == 'darwin'* ]]; then
-	sed -i '' -E 's/^(import.*_pb2)/from . \1/' meshtastic/*.py
-	# automate the current workaround (may be related to Meshtastic-protobufs issue #27 https://github.com/meshtastic/protobufs/issues/27)
-	sed -i '' -E "s/^None = 0/globals()['None'] = 0/" meshtastic/mesh_pb2.py
+	SEDCMD="sed -i '' -E"
 else
-	sed -i -e 's/^import.*_pb2/from . \0/' meshtastic/*.py
-	# automate the current workaround (may be related to Meshtastic-protobufs issue #27 https://github.com/meshtastic/protobufs/issues/27)
-	sed -i -e "s/^None = 0/globals()['None'] = 0/" meshtastic/mesh_pb2.py
+	SEDCMD="sed -i -E"
 fi
+
+
+# change the package names to meshtastic.protobuf
+$SEDCMD 's/^package meshtastic;/package meshtastic.protobuf;/' "${INDIR}/"*.proto
+# fix the imports to match
+$SEDCMD 's/^import "meshtastic\//import "meshtastic\/protobuf\//' "${INDIR}/"*.proto
+
+# Generate the python files
+./nanopb-0.4.8/generator-bin/protoc -I=$TMPDIR/in --python_out "${OUTDIR}" "--mypy_out=${PYIDIR}" $INDIR/*.proto
+
+# Change "from meshtastic.protobuf import" to "from . import"
+$SEDCMD 's/^from meshtastic.protobuf import/from . import/' "${OUTDIR}"/meshtastic/protobuf/*pb2*.py[i]
+
+# Create a __init__.py in the out directory
+touch "${OUTDIR}/meshtastic/protobuf/__init__.py"
+
+# Copy to the source controlled tree
+mkdir -p meshtastic/protobuf
+rm -rf meshtastic/protobuf/*pb2*.py
+cp "${OUTDIR}/meshtastic/protobuf"/* meshtastic/protobuf
+
+exit 0

meshtastic/__init__.py

@@ -81,7 +81,10 @@ def onConnection(interface, topic=pub.AUTO_TOPIC): # called when we (re)connect
 from pubsub import pub # type: ignore[import-untyped]
 from tabulate import tabulate
 
-from meshtastic import (
+from meshtastic.node import Node
+from meshtastic.util import DeferredExecution, Timeout, catchAndIgnore, fixme, stripnl
+
+from .protobuf import (
     admin_pb2,
     apponly_pb2,
     channel_pb2,
@@ -93,10 +96,10 @@ def onConnection(interface, topic=pub.AUTO_TOPIC): # called when we (re)connect
     remote_hardware_pb2,
     storeforward_pb2,
     telemetry_pb2,
+)
+from . import (
     util,
 )
-from meshtastic.node import Node
-from meshtastic.util import DeferredExecution, Timeout, catchAndIgnore, fixme, stripnl
 
 # Note: To follow PEP224, comments should be after the module variable.
 

meshtastic/__main__.py

@@ -17,7 +17,8 @@
 import meshtastic.test
 import meshtastic.util
 from meshtastic import mt_config
-from meshtastic import channel_pb2, config_pb2, portnums_pb2, remote_hardware, BROADCAST_ADDR
+from meshtastic.protobuf import channel_pb2, config_pb2, portnums_pb2
+from meshtastic import remote_hardware, BROADCAST_ADDR
 from meshtastic.version import get_active_version
 from meshtastic.ble_interface import BLEInterface
 from meshtastic.mesh_interface import MeshInterface

meshtastic/mesh_interface.py

@@ -18,10 +18,13 @@
 from tabulate import tabulate
 
 import meshtastic.node
-from meshtastic import (
+
+from meshtastic.protobuf import (
     mesh_pb2,
     portnums_pb2,
     telemetry_pb2,
+)
+from meshtastic import (
     BROADCAST_ADDR,
     BROADCAST_NUM,
     LOCAL_ADDR,

meshtastic/node.py

@@ -7,7 +7,7 @@
 
 from typing import Union
 
-from meshtastic import admin_pb2, apponly_pb2, channel_pb2, localonly_pb2, mesh_pb2, portnums_pb2
+from meshtastic.protobuf import admin_pb2, apponly_pb2, channel_pb2, localonly_pb2, mesh_pb2, portnums_pb2
 from meshtastic.util import (
     Timeout,
     camel_to_snake,

meshtastic/remote_hardware.py

@@ -4,7 +4,7 @@
 
 from pubsub import pub # type: ignore[import-untyped]
 
-from meshtastic import portnums_pb2, remote_hardware_pb2
+from meshtastic.protobuf import portnums_pb2, remote_hardware_pb2
 from meshtastic.util import our_exit
 
 

meshtastic/tests/test_main.py

@@ -21,7 +21,7 @@
 )
 from meshtastic import mt_config
 
-from ..channel_pb2 import Channel # pylint: disable=E0611
+from ..protobuf.channel_pb2 import Channel # pylint: disable=E0611
 
 # from ..ble_interface import BLEInterface
 from ..node import Node

meshtastic/tests/test_mesh_interface.py

@@ -7,7 +7,8 @@
 import pytest
 from hypothesis import given, strategies as st
 
-from .. import mesh_pb2, config_pb2, BROADCAST_ADDR, LOCAL_ADDR
+from ..protobuf import mesh_pb2, config_pb2
+from .. import BROADCAST_ADDR, LOCAL_ADDR
 from ..mesh_interface import MeshInterface, _timeago
 from ..node import Node
 

meshtastic/tests/test_node.py

@@ -6,8 +6,8 @@
 
 import pytest
 
-from .. import localonly_pb2, config_pb2
-from ..channel_pb2 import Channel # pylint: disable=E0611
+from ..protobuf import localonly_pb2, config_pb2
+from ..protobuf.channel_pb2 import Channel # pylint: disable=E0611
 from ..node import Node
 from ..serial_interface import SerialInterface
 from ..mesh_interface import MeshInterface