Fix the sdcard corruption when file is deleted, add permanently motio…

…n detection, full support for M5stack and esp32one, add console window colorization with blinking cursor

modules/lib/motion/historic.py

@@ -40,7 +40,7 @@ def get_root():
 		return None
 
 	@staticmethod
-	async def add_motion(path, name, image, info):
+	async def add_motion(path, name, image, motion_info):
 		""" Add motion detection in the historic """
 		root = Historic.get_root()
 		result = False
@@ -49,7 +49,7 @@ async def add_motion(path, name, image, info):
 				await Historic.acquire()
 				path = strings.tostrings(path)
 				name = strings.tostrings(name)
-				item = Historic.create_item(root + "/" + path + "/" + name +".json", info)
+				item = Historic.create_item(root + "/" + path + "/" + name +".json", motion_info)
 				res1 = sdcard.SdCard.save(path, name + ".jpg" , image)
 				res2 = sdcard.SdCard.save(path, name + ".json", json.dumps(item))
 				Historic.add_item(item)
@@ -61,13 +61,13 @@ async def add_motion(path, name, image, info):
 		return result
 
 	@staticmethod
-	def create_item(filename, info):
+	def create_item(filename, motion_info):
 		""" Create historic item """
 		name = filesystem.splitext(filename)[0] + ".jpg"
 		result = None
-		if "geometry" in info:
+		if "geometry" in motion_info:
 			# Add json file to the historic
-			result = [name, info["geometry"]["width"],info["geometry"]["height"], info["diff"]["diffs"], info["diff"]["squarex"], info["diff"]["squarey"]]
+			result = [name, motion_info["geometry"]["width"],motion_info["geometry"]["height"], motion_info["diff"]["diffs"], motion_info["diff"]["squarex"], motion_info["diff"]["squarey"]]
 		return result
 
 	@staticmethod
@@ -93,12 +93,13 @@ async def build(motions):
 				# For all motions
 				for motion in motions:
 					path = root + "/" + motion
-
 					try:
 						# Parse json file
 						file = None
 						file = open(motion, "rb")
-						Historic.add_item(json.load(file))
+						motion_item = json.load(file)
+						if filesystem.exists(motion_item[0]):
+							Historic.add_item(motion_item)
 					except Exception as err:
 						logger.syslog(err)
 					finally:
@@ -223,60 +224,47 @@ async def remove_files(directory, simulate=False, force=False):
 		""" Remove all files in the directory """
 		import shell
 		dir_not_empty = False
+		enough_space = False
 		force = True
 		if filesystem.exists(directory):
-			# Parse all directories in sdcard
+			files_to_remove = []
+			dirs_to_remove  = []
+
+			# Parse all directories in sdcard (WARNING : TO AVOID CRASH, NEVER DELETE DIRECTORY OR FILE IN ilistdir LOOP)
 			for fileinfo in uos.ilistdir(directory):
 				filename = fileinfo[0]
 				typ      = fileinfo[1]
 				# If file found
 				if typ & 0xF000 != 0x4000:
-					shell.rmfile(directory + "/" + filename, simulate=simulate, force=force)
+					files_to_remove.append(directory + "/" + filename)
 				else:
-					await Historic.remove_files(directory + "/" + filename, simulate=simulate, force=force)
+					dirs_to_remove.append(directory + "/" + filename)
+				dir_not_empty = True
 
-					# Force the parsing of historic
-					Historic.first_extract[0] = False
-					dir_not_empty = True
-
-				if Historic.is_not_enough_space(low=False) is False:
+			for file_to_remove in files_to_remove:
+				shell.rmfile(file_to_remove, simulate=simulate, force=force)
+				if sdcard.SdCard.is_not_enough_space(low=False) is False:
+					enough_space = True
 					break
 
-			if dir_not_empty:
-				shell.rm(directory, recursive=True, simulate=simulate, force=force)
-			else:
-				shell.rmdir(directory, recursive=True, simulate=simulate, force=force)
-		else:
-			print("Directory not existing '%s'"%directory)
-
-	@staticmethod
-	def is_not_enough_space(low):
-		""" Indicates if remaining space is not sufficient """
-		free = sdcard.SdCard.get_free_size()
-		total = sdcard.SdCard.get_max_size()
-		if low:
-			if sdcard.SdCard.is_available():
-				threshold = 5
-			else:
-				threshold = 5
-		else:
-			if sdcard.SdCard.is_available():
-				threshold = 8
-			else:
-				threshold = 25
-
-		if free < 0 or total < 0:
-			return True
-		return ((free * 100 // total) <= threshold)
+			if enough_space is False:
+				for dir_to_remove in dirs_to_remove:
+					await Historic.remove_files(dir_to_remove, simulate=simulate, force=force)
+				if dir_not_empty:
+					shell.rm(directory, recursive=True, simulate=simulate, force=force)
+				else:
+					shell.rmdir(directory, recursive=True, simulate=simulate, force=force)
 
 	@staticmethod
 	async def remove_older(force=False):
 		""" Remove older files to make space """
 		root = Historic.get_root()
 		if root:
 			# If not enough space available on sdcard
-			if Historic.is_not_enough_space(low=True) or force:
-				logger.syslog("Start cleanup sd card")
+			if sdcard.SdCard.is_not_enough_space(low=True) or force:
+				logger.syslog("Start cleanup historic")
+				cleaned = "Historic space after cleanup"
+				Historic.first_extract[0] = False
 				olders = await Historic.scan_directories(MAX_REMOVED, True)
 				previous = ""
 				for motion in olders:
@@ -290,23 +278,22 @@ async def remove_older(force=False):
 						logger.syslog(err)
 					finally:
 						await Historic.release()
-					if Historic.is_not_enough_space(low=False) is False:
-						logger.syslog("Sd card has enough space")
+					if sdcard.SdCard.is_not_enough_space(low=False) is False:
 						break
-				logger.syslog("End cleanup sd card")
-				logger.syslog(strings.tostrings(info.flashinfo(mountpoint=sdcard.SdCard.get_mountpoint(), display=False)))
+				logger.syslog("End cleanup historic")
+			else:
+				cleaned = "Historic space"
+			logger.syslog("%s : %s"%(cleaned, strings.tostrings(info.flashinfo(mountpoint=sdcard.SdCard.get_mountpoint(), display=False))))
 
 	@staticmethod
 	async def periodic():
 		""" Internal periodic task """
 		from server.server import Server
 		if filesystem.ismicropython():
-			if sdcard.SdCard.is_available():
-				await Server.wait_resume(307)
-			else:
-				await Server.wait_resume(71)
+			await Server.wait_resume(307)
 		else:
-			await Server.wait_resume(7)
+			await Server.wait_resume(17)
+
 		if Historic.motion_in_progress[0] is False:
 			if sdcard.SdCard.is_mounted():
 				await Historic.remove_older()

modules/lib/motion/motion.py

@@ -50,6 +50,10 @@ def __init__(self):
 		# Notify motion state change
 		self.notify_state = True
 
+		# Permanent detection without notification.
+		# To keep all motion detection in the presence of occupants
+		self.permanent_detection = False
+
 		# Empty mask is equal disable masking
 		self.mask = b""
 
@@ -523,9 +527,10 @@ async def detect(self):
 		self.release_image()
 
 		# If the motion detection activated
-		if await self.is_activated():
+		activated = await self.is_activated()
+		if activated or self.is_pemanent():
 			# Capture motion
-			result = await self.capture()
+			result = await self.capture(activated)
 		else:
 			if self.motion:
 				self.motion.stop_light()
@@ -576,6 +581,16 @@ async def is_activated(self):
 			await uasyncio.sleep_ms(500)
 		return result
 
+	def is_pemanent(self):
+		""" Indicates if pemanent detection activated """
+		result = False
+		# If motion activated
+		if self.motion_config.activated:
+			# If detection permanent without notification activated
+			if self.motion_config.permanent_detection:
+				result = True
+		return result
+
 	async def init_motion(self):
 		""" Initialize motion detection """
 		firstInit = False
@@ -608,7 +623,7 @@ def release_image(self):
 			if self.motion.index %30 == 0:
 				collect()
 
-	async def capture(self):
+	async def capture(self, activated):
 		""" Capture motion """
 		result = False
 
@@ -628,8 +643,8 @@ async def capture(self):
 				# Capture motion image
 				self.detection = await self.motion.capture()
 
-				# If motion detected
-				if self.detection is not None:
+				# If motion detected and detection activated
+				if self.detection is not None and activated is True:
 					# Notify motion with push over
 					message, image = self.detection
 					if self.motion_config.notify:

modules/lib/shell/editor.py

@@ -880,7 +880,7 @@ def replace_char(self, char):
 	def open_selection(self):
 		""" Start a selection """
 		if self.selection_start is None:
-			self.selection_start = [self.cursor_column, self.cursor_line, self.get_tab_cursor(self.cursor_line)]
+			self.selection_start = [self.cursor_column, self.cursor_line, self.get_tab_cursor(self.cursor_line, self.cursor_column)]
 
 	def close_selection(self):
 		""" Terminate selection """

modules/lib/tools/builddate.py

@@ -1 +1 @@
-date=b'2022/03/27  20:11:06'
+date=b'2022/04/08  18:02:01'

modules/lib/tools/lang_english.py

@@ -1,3 +1,4 @@
+""" English texts """
 # Distributed under MIT License
 # Copyright (c) 2021 Remi BERTHOLET
 
@@ -57,6 +58,7 @@
 notification_motion                     =b"Notify motion detection"
 notification_state                      =b"Notify state change"
 suspends_motion_detection               =b"Suspends motion detection on the presence of an occupant"
+permanent_detection                     =b"Permanently archive all motion detections"
 turn_on_flash                           =b"Turn on the led flash when the light goes down"
 pushover_on                             =b"Pushover notification on"
 pushover_off                            =b"Pushover notification off"

modules/lib/tools/lang_french.py

@@ -1,3 +1,4 @@
+""" Textes en francais """
 # Distributed under MIT License
 # Copyright (c) 2021 Remi BERTHOLET
 
@@ -57,6 +58,7 @@
 notification_motion                     =b"Notifier d\xC3\xA9tection de mouvement"
 notification_state                      =b"Notifier changement \xC3\xA9tat"
 suspends_motion_detection               =b"Suspendre la d\xC3\xA9tection de mouvement en pr\xC3\xA9sence d'occupants"
+permanent_detection                     =b"Archiver en permanence toutes les d\xC3\xA9tection de mouvements"
 turn_on_flash                           =b"Allumer le flash LED lorsque la lumi\xC3\xA8re baisse"
 pushover_on                             =b"Notification pushover activ\xC3\xA9e"
 pushover_off                            =b"Notification pushover d\xC3\xA9sactiv\xC3\xA9e"
@@ -174,4 +176,4 @@
 saturation                              =b"Saturation"
 hmirror                                 =b"Mirroir horizontal"
 vflip                                   =b"Rotation verticale"
-flash_level                             =b"Luminosit\xC3\xA9 du flash"
+flash_level                             =b"Luminosit\xC3\xA9 du flash"

modules/lib/tools/sdcard.py

@@ -152,20 +152,48 @@ def create_file(directory, filename, mode="w"):
 					break
 		return result
 
+	@staticmethod
+	def is_not_enough_space(low):
+		""" Indicates if remaining space is not sufficient """
+		free = SdCard.get_free_size()
+		total = SdCard.get_max_size()
+		if low:
+			if SdCard.is_available():
+				threshold = 5
+			else:
+				threshold = 5
+		else:
+			if SdCard.is_available():
+				threshold = 8
+			else:
+				threshold = 25
+
+		if free < 0 or total < 0:
+			return True
+
+		if free < 32*1024*4:
+			return True
+		else:
+			return ((free * 100 // total) <= threshold)
+
 	@staticmethod
 	def save(directory, filename, data):
 		""" Save file on sd card """
 		result = False
 		if SdCard.is_mounted():
 			file = None
-			try:
-				file = SdCard.create_file(SdCard.get_mountpoint() + "/" + directory, filename, "w")
-				file.write(data)
-				file.close()
-				result = True
-			except Exception as err:
-				logger.syslog(err, "Cannot save %s/%s/%s"%(SdCard.get_mountpoint(), directory, filename))
-			finally:
-				if file is not None:
+			if SdCard.is_not_enough_space(low=True) is False:
+				try:
+					file = SdCard.create_file(SdCard.get_mountpoint() + "/" + directory, filename, "w")
+					file.write(data)
 					file.close()
+					result = True
+				except Exception as err:
+					logger.syslog(err, "Cannot save %s/%s/%s"%(SdCard.get_mountpoint(), directory, filename))
+				finally:
+					if file is not None:
+						file.close()
+			else:
+				if SdCard.is_available() is False:
+					result = True
 		return result

modules/lib/webpage/motionpage.py

@@ -153,7 +153,8 @@ async def motion(request, response, args):
 		Switch(text=lang.notification_motion, name=b"notify",       checked=config.notify,       disabled=disabled),Br(),
 		Switch(text=lang.notification_state,  name=b"notify_state", checked=config.notify_state, disabled=disabled),Br(),
 		Switch(text=lang.suspends_motion_detection,                name=b"suspend_on_presence", checked=config.suspend_on_presence, disabled=disabled),Br(),
-		Switch(text=lang.turn_on_flash,                            name=b"light_compensation", checked=config.light_compensation, disabled=disabled),Br(),
+		Switch(text=lang.permanent_detection,                      name=b"permanent_detection", checked=config.permanent_detection, disabled=disabled),Br(),
+		Switch(text=lang.turn_on_flash,                            name=b"light_compensation",  checked=config.light_compensation,  disabled=disabled),Br(),
 		submit)
 	await response.send_page(page)
 

tools/camflasher/camflasher.py

@@ -19,20 +19,25 @@ def setupUi(self, CamFlasher):
         self.main_layout = QtWidgets.QWidget(CamFlasher)
         self.main_layout.setObjectName("main_layout")
         self.gridLayout = QtWidgets.QGridLayout(self.main_layout)
+        self.gridLayout.setContentsMargins(3, 3, 3, 3)
+        self.gridLayout.setSpacing(0)
         self.gridLayout.setObjectName("gridLayout")
+        self.horizontalLayout = QtWidgets.QHBoxLayout()
+        self.horizontalLayout.setContentsMargins(-1, -1, -1, 0)
+        self.horizontalLayout.setObjectName("horizontalLayout")
         self.label_port = QtWidgets.QLabel(self.main_layout)
         sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Policy.Fixed, QtWidgets.QSizePolicy.Policy.Preferred)
         sizePolicy.setHorizontalStretch(0)
         sizePolicy.setVerticalStretch(0)
         sizePolicy.setHeightForWidth(self.label_port.sizePolicy().hasHeightForWidth())
         self.label_port.setSizePolicy(sizePolicy)
         self.label_port.setObjectName("label_port")
-        self.gridLayout.addWidget(self.label_port, 0, 0, 1, 1)
+        self.horizontalLayout.addWidget(self.label_port)
         self.combo_port = QtWidgets.QComboBox(self.main_layout)
         self.combo_port.setEnabled(True)
         self.combo_port.setFocusPolicy(QtCore.Qt.FocusPolicy.NoFocus)
         self.combo_port.setObjectName("combo_port")
-        self.gridLayout.addWidget(self.combo_port, 0, 1, 1, 1)
+        self.horizontalLayout.addWidget(self.combo_port)
         self.chk_rts_dtr = QtWidgets.QCheckBox(self.main_layout)
         sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Policy.Maximum, QtWidgets.QSizePolicy.Policy.Fixed)
         sizePolicy.setHorizontalStretch(0)
@@ -41,7 +46,8 @@ def setupUi(self, CamFlasher):
         self.chk_rts_dtr.setSizePolicy(sizePolicy)
         self.chk_rts_dtr.setFocusPolicy(QtCore.Qt.FocusPolicy.NoFocus)
         self.chk_rts_dtr.setObjectName("chk_rts_dtr")
-        self.gridLayout.addWidget(self.chk_rts_dtr, 0, 2, 1, 1)
+        self.horizontalLayout.addWidget(self.chk_rts_dtr)
+        self.gridLayout.addLayout(self.horizontalLayout, 0, 0, 1, 1)
         self.output = QtWidgets.QTextBrowser(self.main_layout)
         self.output.setEnabled(True)
         font = QtGui.QFont()
@@ -50,7 +56,7 @@ def setupUi(self, CamFlasher):
         self.output.viewport().setProperty("cursor", QtGui.QCursor(QtCore.Qt.CursorShape.ArrowCursor))
         self.output.setReadOnly(False)
         self.output.setObjectName("output")
-        self.gridLayout.addWidget(self.output, 1, 0, 1, 3)
+        self.gridLayout.addWidget(self.output, 1, 0, 1, 1)
         CamFlasher.setCentralWidget(self.main_layout)
         self.menu_bar = QtWidgets.QMenuBar(CamFlasher)
         self.menu_bar.setGeometry(QtCore.QRect(0, 0, 800, 24))