By clicking “Sign up for GitHub”, you agree to our
terms of service
and
privacy statement
. We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Describe the bug
With a current version of micropython and upip, installing the package places the required files in the appropriate path, but it seems that something in the library management doesn't handle user-installed libraries with the same directory name as built-in libraries properly:
To Reproduce
>>> import upip
>>> import os
>>> import sys
>>> upip.install("micropython-umqtt.robust2")
Installing to: /lib/
Warning: micropython.org SSL certificate is not validated
Installing micropython-umqtt.simple2 2.1.0 from https://files.pythonhosted.org/packages/a7/75/c80449358ec98512393095b8180f4b59fd0ca9582f27fbb837d160da9e6b/micropython-umqtt.simple2-2.1.0.tar.gz
>>> from umqtt.simple2import MQTTClient
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ImportError: no module named 'umqtt.simple2'
>>> sys.path
['', '/lib']
>>> os.listdir("/lib")
['umqtt']
>>> os.listdir("/lib/umqtt")
['__init__.py', 'errno.py', 'robust2.py', 'simple2.py']
>>> open("/lib/umqtt/simple2.py").read()
"import usocket as socket\nimport uselect\nfrom utime import ticks_add,ticks_ms,ticks_diff\nclass MQTTException(Exception):0\ndef pid_gen(pid=0):\n\tA=pid\n\twhile True:A=A+1 if A<65535 else 1;yield A\nclass MQTTClient:\n\tdef __init__(A,client_id,server,port=0,user=None,password=None,keepalive=0,ssl=False,ssl_params=None,socket_timeout=5,message_timeout=10):\n\t\tC=ssl_params;B=port\n\t\tif B==0:B=8883 if ssl else 1883\n\t\tA.client_id=client_id;A.sock=None;A.poller=None;A.server=server;A.port=B;A.ssl=ssl;A.ssl_params=C if C else{};A.newpid=pid_gen()\n\t\tif not getattr(A,'cb',None):A.cb=None\n\t\tif not getattr(A,'cbstat',None):A.cbstat=lambda p,s:None\n\t\tA.user=user;A.pswd=password;A.keepalive=keepalive;A.lw_topic=None;A.lw_msg=None;A.lw_qos=0;A.lw_retain=False;A.rcv_pids={};A.last_ping=ticks_ms();A.last_cpacket=ticks_ms();A.socket_timeout=socket_timeout;A.message_timeout=message_timeout\n\tdef _read(A,n):\n\t\ttry:\n\t\t\tB=b''\n\t\t\tfor C in range(n):A._sock_timeout(A.poller_r,A.socket_timeout);B+=A.sock.read(1)\n\t\texcept AttributeError:raise MQTTException(8)\n\t\tif B==b'':raise MQTTException(1)\n\t\tif len(B)!=n:raise MQTTException(2)\n\t\treturn B\n\tdef _write(A,bytes_wr,length=-1):\n\t\tD=bytes_wr;B=length\n\t\ttry:A._sock_timeout(A.poller_w,A.socket_timeout);C=A.sock.write(D,B)\n\t\texcept AttributeError:raise MQTTException(8)\n\t\tif B<0:\n\t\t\tif C!=len(D):raise MQTTException(3)\n\t\telif C!=B:raise MQTTException(3)\n\t\treturn C\n\tdef _send_str(A,s):assert len(s)<65536;A._write(len(s).to_bytes(2,'big'));A._write(s)\n\tdef _recv_len(D):\n\t\tA=0;B=0\n\t\twhile 1:\n\t\t\tC=D._read(1)[0];A|=(C&127)<<B\n\t\t\tif not C&128:return A\n\t\t\tB+=7\n\tdef _varlen_encode(C,value,buf,offset=0):\n\t\tB=offset;A=value;assert A<268435456\n\t\twhile A>127:buf[B]=A&127|128;A>>=7;B+=1\n\t\tbuf[B]=A;return B+1\n\tdef _sock_timeout(B,poller,socket_timeout):\n\t\tA=socket_timeout\n\t\tif B.sock:\n\t\t\tC=poller.poll(-1 if A is None else int(A*1000))\n\t\t\tif not C:raise MQTTException(30)\n\t\telse:raise MQTTException(28)\n\tdef set_callback(A,f):A.cb=f\n\tdef set_callback_status(A,f):A.cbstat=f\n\tdef set_last_will(A,topic,msg,retain=False,qos=0):B=topic;assert 0<=qos<=2;assert B;A.lw_topic=B;A.lw_msg=msg;A.lw_qos=qos;A.lw_retain=retain\n\tdef connect(A,clean_session=True):\n\t\tE=clean_session;A.sock=socket.socket();A.poller_r=uselect.poll();A.poller_r.register(A.sock,uselect.POLLIN);A.poller_w=uselect.poll();A.poller_w.register(A.sock,uselect.POLLOUT);G=socket.getaddrinfo(A.server,A.port)[0][-1];A.sock.connect(G)\n\t\tif A.ssl:import ussl;A.sock=ussl.wrap_socket(A.sock,**A.ssl_params)\n\t\tF=bytearray(b'\\x10\\x00\\x00\\x00\\x00\\x00');B=bytearray(b'\\x00\\x04MQTT\\x04\\x00\\x00\\x00');D=10+2+len(A.client_id);B[7]=bool(E)<<1\n\t\tif bool(E):A.rcv_pids.clear()\n\t\tif A.user is not None:\n\t\t\tD+=2+len(A.user);B[7]|=1<<7\n\t\t\tif A.pswd is not None:D+=2+len(A.pswd);B[7]|=1<<6\n\t\tif A.keepalive:assert A.keepalive<65536;B[8]|=A.keepalive>>8;B[9]|=A.keepalive&255\n\t\tif A.lw_topic:D+=2+len(A.lw_topic)+2+len(A.lw_msg);B[7]|=4|(A.lw_qos&1)<<3|(A.lw_qos&2)<<3;B[7]|=A.lw_retain<<5\n\t\tH=A._varlen_encode(D,F,1);A._write(F,H);A._write(B);A._send_str(A.client_id)\n\t\tif A.lw_topic:A._send_str(A.lw_topic);A._send_str(A.lw_msg)\n\t\tif A.user is not None:\n\t\t\tA._send_str(A.user)\n\t\t\tif A.pswd is not None:A._send_str(A.pswd)\n\t\tC=A._read(4)\n\t\tif not(C[0]==32 and C[1]==2):raise MQTTException(29)\n\t\tif C[3]!=0:\n\t\t\tif 1<=C[3]<=5:raise MQTTException(20+C[3])\n\t\t\telse:raise MQTTException(20,C[3])\n\t\tA.last_cpacket=ticks_ms();return C[2]&1\n\tdef disconnect(A):A._write(b'\\xe0\\x00');A.poller_r.unregister(A.sock);A.poller_w.unregister(A.sock);A.sock.close();A.sock=None;A.poller=None\n\tdef ping(A):A._write(b'\\xc0\\x00');A.last_ping=ticks_ms()\n\tdef publish(A,topic,msg,retain=False,qos=0,dup=False):\n\t\tE=topic;B=qos;assert B in(0,1);C=bytearray(b'0\\x00\\x00\\x00\\x00');C[0]|=B<<1|retain|int(dup)<<3;F=2+len(E)+len(msg)\n\t\tif B>0:F+=2\n\t\tG=A._varlen_encode(F,C,1);A._write(C,G);A._send_str(E)\n\t\tif B>0:D=next(A.newpid);A._write(D.to_bytes(2,'big'))\n\t\tA._write(msg)\n\t\tif B>0:A.rcv_pids[D]=ticks_add(ticks_ms(),A.message_timeout*1000);return D\n\tdef subscribe(A,topic,qos=0):E=topic;assert qos in(0,1);assert A.cb is not None,'Subscribe callback is not set';B=bytearray(b'\\x82\\x00\\x00\\x00\\x00\\x00\\x00');C=next(A.newpid);F=2+2+len(E)+1;D=A._varlen_encode(F,B,1);B[D:D+2]=C.to_bytes(2,'big');A._write(B,D+2);A._send_str(E);A._write(qos.to_bytes(1,'little'));A.rcv_pids[C]=ticks_add(ticks_ms(),A.message_timeout*1000);return C\n\tdef _message_timeout(A):\n\t\tC=ticks_ms()\n\t\tfor (B,D) in A.rcv_pids.items():\n\t\t\tif ticks_diff(D,C)<=0:A.rcv_pids.pop(B);A.cbstat(B,0)\n\tdef check_msg(A):\n\t\tif A.sock:\n\t\t\tif not A.poller_r.poll(-1 if A.socket_timeout is None else 1):A._message_timeout();return None\n\t\t\ttry:\n\t\t\t\tG=A._read(1)\n\t\t\t\tif not G:A._message_timeout();return None\n\t\t\texcept OSError as H:\n\t\t\t\tif H.args[0]==110:A._message_timeout();return None\n\t\t\t\telse:raise H\n\t\telse:raise MQTTException(28)\n\t\tif G==b'\\xd0':\n\t\t\tif A._read(1)[0]!=0:MQTTException(-1)\n\t\t\tA.last_cpacket=ticks_ms();return\n\t\tB=G[0]\n\t\tif B==64:\n\t\t\tD=A._read(1)\n\t\t\tif D!=b'\\x02':raise MQTTException(-1)\n\t\t\tF=int.from_bytes(A._read(2),'big')\n\t\t\tif F in A.rcv_pids:A.last_cpacket=ticks_ms();A.rcv_pids.pop(F);A.cbstat(F,1)\n\t\t\telse:A.cbstat(F,2)\n\t\tif B==144:\n\t\t\tC=A._read(4)\n\t\t\tif C[0]!=3:raise MQTTException(40,C)\n\t\t\tif C[3]==128:raise MQTTException(44)\n\t\t\tif C[3]not in(0,1,2):raise MQTTException(40,C)\n\t\t\tE=C[2]|C[1]<<8\n\t\t\tif E in A.rcv_pids:A.last_cpacket=ticks_ms();A.rcv_pids.pop(E);A.cbstat(E,1)\n\t\t\telse:raise MQTTException(5)\n\t\tA._message_timeout()\n\t\tif B&240!=48:return B\n\t\tD=A._recv_len();I=int.from_bytes(A._read(2),'big');J=A._read(I);D-=I+2\n\t\tif B&6:E=int.from_bytes(A._read(2),'big');D-=2\n\t\tK=A._read(D)if D else b'';L=B&1;M=B&8;A.cb(J,K,bool(L),bool(M));A.last_cpacket=ticks_ms()\n\t\tif B&6==2:A._write(b'@\\x02');A._write(E.to_bytes(2,'big'))\n\t\telif B&6==4:raise NotImplementedError()\n\t\telif B&6==6:raise MQTTException(-1)\n\tdef wait_msg(A):B=A.socket_timeout;A.socket_timeout=None;C=A.check_msg();A.socket_timeout=B;return C"
>>> os.listdir("/lib")
['umqtt']
>>> os.listdir("/lib/umqtt")
['__init__.py', 'errno.py', 'robust2.py', 'simple2.py']
>>> os.rename("/lib/umqtt", "/lib/umqtt2")
>>> from umqtt2.simple2 import MQTTClient
Steps to reproduce the behavior:
On a blank ESP32 (or similar) with network access and upython 1.13.0.
1: Install package
2: Try to import.
Renaming the
umqtt
directory that
upip
places in
/lib
to something else (I used
umqtt2
) lets it import correctly. My current guess is that the import system doesn't handle user libraries with the same name as baked in libraries.
Details (please complete the following information):
ESP32
Micropython version:
sys.version
'3.4.0'
sys.implementation
(name='micropython', version=(1, 13, 0), mpy=10757)
Currently using the official download
esp32-idf3-20200902-v1.13.bin
from
https://micropython.org/download/esp32/
Please check again. Because I can't confirm this problem.
I tested on ESP32 and Micropython 1.13.
Look below...
>>> import upip
>>> import os
>>> import sys
>>> sys.path
['', '/lib']
>>> os.listdir("/lib")
['mcron', 'urequests.mpy']
>>> os.listdir("/lib/umqtt")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
OSError: [Errno 2] ENOENT
>>> from umqtt.simple2import MQTTClient
Traceback (most recent call last):
File "<stdin>", line 1
SyntaxError: invalid syntax
>>> upip.install("micropython-umqtt.simple2")
Installing to: /lib/
Warning: micropython.org SSL certificate is not validated
Installing micropython-umqtt.simple2 2.1.0 from https://files.pythonhosted.org/packages/a7/75/c80449358ec98512393095b8180f4b59fd0ca9582f27fbb837d160da9e6b/micropython-umqtt.simple2-2.1.0.tar.gz
>>> upip.install("micropython-umqtt.robust2")
Installing to: /lib/
Installing micropython-umqtt.robust2 2.1.0 from https://files.pythonhosted.org/packages/2a/f1/5f50372df69322fc82d35d9248b37532ffc0046aa9758095c8297ea5e7ca/micropython-umqtt.robust2-2.1.0.tar.gz
>>> sys.path
['', '/lib']
>>> os.listdir("/lib")
['mcron', 'umqtt', 'urequests.mpy']
>>> os.listdir("/lib/umqtt")
['__init__.py', 'errno.py', 'robust2.py', 'simple2.py']
>>> from umqtt.simple2 import MQTTClient
>>> MQTTClient.__class__
<class 'type'>
>>> sys.version
'3.4.0'
>>> sys.implementation
(name='micropython', version=(1, 13, 0), mpy=10757)
Beats me. I tried on a different esp32 with a reset flash, and I'm still getting the failure:
Script to replicate: https://gist.github.com/fake-name/e09d4eb1f67fea721c5209af6b13804a
You will need a separate file creds.py which contains two values, network and password containing the utf-8 encoded SSID of your WiFi and pass-key respectively. Yes, my WiFi is named "(╯°□°)╯︵ ┻━┻" (as an aside, this is an interesting, if annoying way to discover all the micropython tools that fail completely to handle non-ascii file contents. It can break uPyCraft somehow, for example).
I did the following test by erasing the esp32 flash, programming it with esp32-idf3-20200902-v1.13.bin, using ampy to upload boot.py and creds.py, and then issuing import machine; machine.reset() in the repl.
Complete stdout log (sorry about the messy VT-100 command codes. uPyCraft really doesn't like non-ascii).
>>> import machine
>>> machine.reset()
ets Jun 8 2016 00:22:57
rst:0xc (SW_CPU_RESET),boot:0x13 (SPI_FAST_FLASH_BOOT)
configsip: 0, SPIWP:0xee
clk_drv:0x00,q_drv:0x00,d_drv:0x00,cs0_drv:0x00,hd_drv:0x00,wp_drv:0x00
mode:DIO, clock div:2
load:0x3fff0018,len:4
load:0x3fff001c,len:5008
ho 0 tail 12 room 4
load:0x40078000,len:10600
ho 0 tail 12 room 4
load:0x40080400,len:5684
entry 0x400806bc
�[0;32mI (539) cpu_start: Pro cpu up.�[0m
�[0;32mI (539) cpu_start: Application information:�[0m
�[0;32mI (539) cpu_start: Compile time: Sep 2 2020 03:00:08�[0m
�[0;32mI (542) cpu_start: ELF file SHA256: 0000000000000000...�[0m
�[0;32mI (548) cpu_start: ESP-IDF: v3.3.2�[0m
�[0;32mI (553) cpu_start: Starting app cpu, entry point is 0x40082f30�[0m
�[0;32mI (544) cpu_start: App cpu up.�[0m
�[0;32mI (564) heap_init: Initializing. RAM available for dynamic allocation:�[0m
�[0;32mI (571) heap_init: At 3FFAFF10 len 000000F0 (0 KiB): DRAM�[0m
�[0;32mI (577) heap_init: At 3FFB6388 len 00001C78 (7 KiB): DRAM�[0m
�[0;32mI (583) heap_init: At 3FFB9A20 len 00004108 (16 KiB): DRAM�[0m
�[0;32mI (589) heap_init: At 3FFBDB5C len 00000004 (0 KiB): DRAM�[0m
�[0;32mI (595) heap_init: At 3FFCA9E8 len 00015618 (85 KiB): DRAM�[0m
�[0;32mI (601) heap_init: At 3FFE0440 len 00003AE0 (14 KiB): D/IRAM�[0m
�[0;32mI (608) heap_init: At 3FFE4350 len 0001BCB0 (111 KiB): D/IRAM�[0m
�[0;32mI (614) heap_init: At 4009DE28 len 000021D8 (8 KiB): IRAM�[0m
�[0;32mI (620) cpu_start: Pro cpu start user code�[0m
�[0;32mI (303) cpu_start: Starting scheduler on PRO CPU.�[0m
�[0;32mI (0) cpu_start: Starting scheduler on APP CPU.�[0m
Booting....
I (170) wifi:wifi driver task: 3ffd1000, prio:23, stack:3584, core=0
�[0;32mI (646) system_api: Base MAC address is not set, read default base MAC address from BLK0 of EFUSE�[0m
�[0;32mI (646) system_api: Base MAC address is not set, read default base MAC address from BLK0 of EFUSE�[0m
I (706) wifi:wifi firmware version: 44aa95c
I (706) wifi:config NVS flash: enabled
I (706) wifi:config nano formating: disabled
I (706) wifi:Init dynamic tx buffer num: 32
I (716) wifi:Init data frame dynamic rx buffer num: 32
I (716) wifi:Init management frame dynamic rx buffer num: 32
I (726) wifi:Init management short buffer num: 32
I (726) wifi:Init static rx buffer size: 1600
I (736) wifi:Init static rx buffer num: 10
I (736) wifi:Init dynamic rx buffer num: 32
�[0;33mW (746) phy_init: failed to load RF calibration data (0x1102), falling back to full calibration�[0m
�[0;32mI (896) phy: phy_version: 4180, cb3948e, Sep 12 2019, 16:39:13, 0, 2�[0m
I (906) wifi:mode : sta (24:6f:28:79:cc:6c)
�[0;32mI (916) wifi: STA_START�[0mE
abling WiFi True
Availble networks:
�[0;32mI (2966) network: event 1�[0m
Connecting to <module 'creds' from 'creds.py'>
Waiting for wifi 30....
I (3136) wifi:new:<1,0>, old:<1,0>, ap:<255,255>, sta:<1,0>, prof:1
I (3986) wifi:state: init -> auth (b0)
I (3986) wifi:state: auth -> assoc (0)
I (3996) wifi:state: assoc -> run (10)
I (4006) wifi:connected with (╯°□°)╯︵ ┻━��, aid = 5, channel 1, BW20, bssid = 80:2a:a8:57:f9:3b
I (4006) wifi:security type: 3, phy: bgn, rssi: -56
I (4006) wifi:pm start, type: 1
�[0;32mI (4016) network: CONNECTED�[0m
I (4026) wifi:AP's beacon interval = 102400 us, DTIM period = 3
Waiting for wifi 29....
�[0;32mI (4596) event: sta ip: 10.1.2.147, mask: 255.255.0.0, gw: 10.1.1.1�[0m
�[0;32mI (4596) network: GOT_IP�[0m
Connected!
micropython-umqtt.simple2 Library not installed. Installing and rebooting
�[0;32mI (5046) modsocket: Initializing�[0m
Installing to: /lib/
Warning: micropython.org SSL certificate is not validated
Installing micropython-umqtt.simple2 2.1.0 from https://files.pythonhosted.org/packages/a7/75/c80449358ec98512393095b8180f4b59fd0ca9582f27fbb837d160da9e6b/micropython-umqtt.simple2-2.1.0.tar.gz
Installing to: /lib/
Installing micropython-umqtt.robust2 2.1.0 from https://files.pythonhosted.org/packages/2a/f1/5f50372df69322fc82d35d9248b37532ffc0046aa9758095c8297ea5e7ca/micropython-umqtt.robust2-2.1.0.tar.gz
Importing installed file
Traceback (most recent call last):
File "boot.py", line 80, in <module>
ImportError: no module named 'umqtt.robust2'
MicroPython v1.13 on 2020-09-02; ESP32 module with ESP32
Type "help()" for more information.
Actually there is a problem with the firmware downloaded from the site:
https://micropython.org/download/esp32/
https://micropython.org/download/esp8266/ .
The reason is that the firmware already contains the original version of umqtt from the micropython-lib repository.
Micropython looking for the library first finds the umqtt "directory" located in the firmware, and there it looks for more files to import. This is how python works.
Solutions:
we compile the appropriate version of Micropython ourselves from source.
either in this repository we change the name from umqtt to umqtt2
The reason is that the firmware already contains the original version of umqtt from the micropython-lib repository.
Isn't that literally exactly what my original bug report said?
Anyways, the root of the problem seems to be that micropython now ships the umqtt library as part of the stdlib, and quoting from the original bug report:
My current guess is that the import system doesn't handle user libraries with the same name as baked in libraries.
Yes, yes.
I have added information to the readme.rst file on how to work around this problem.
Thank you for reporting this problem. Actually the problem itself is not directly related to this library, but any other library that will overwrite some firmware library.
It'd probably be worth considering changing the package name, since it seems like upstream micropython isn't going to revert these changes at any point.
I'd kind of suspect that globally reversing the import order could have other side effects, it doesn't seem like a great idea.
