You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

195 lines
5.0 KiB

10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
10 years ago
  1. #! /usr/bin/env python
  2. import subprocess
  3. from time import sleep
  4. from threading import Thread
  5. from wx import CallAfter
  6. from wx.lib.pubsub import setuparg1
  7. from wx.lib.pubsub import pub as Publisher
  8. from .OutputHandler import (
  9. DataPack,
  10. OutputFormatter
  11. )
  12. from .Utils import (
  13. get_encoding,
  14. encode_list,
  15. remove_file,
  16. get_os_type,
  17. file_exist
  18. )
  19. MAX_DOWNLOAD_THREADS = 3
  20. PUBLISHER_TOPIC = 'download'
  21. class DownloadManager(Thread):
  22. def __init__(self, options, downloadlist, clear_dash_files, logmanager=None):
  23. super(DownloadManager, self).__init__()
  24. self.clear_dash_files = clear_dash_files
  25. self.downloadlist = downloadlist
  26. self.logmanager = logmanager
  27. self.options = options
  28. self.running = True
  29. self.procList = []
  30. self.procNo = 0
  31. self.start()
  32. def run(self):
  33. while self.running:
  34. if self.downloadlist:
  35. # Extract url, index from data
  36. url, index = self.extract_data()
  37. # Wait for your turn if there are not more positions in 'queue'
  38. while self.procNo >= MAX_DOWNLOAD_THREADS:
  39. proc = self.check_queue()
  40. if proc != None:
  41. self.procList.remove(proc)
  42. self.procNo -= 1
  43. sleep(1)
  44. # If we still running create new ProcessWrapper thread
  45. if self.running:
  46. self.procList.append(
  47. ProcessWrapper(
  48. self.options,
  49. url,
  50. index,
  51. self.clear_dash_files,
  52. self.logmanager
  53. )
  54. )
  55. self.procNo += 1
  56. else:
  57. # Return True if at least one process is alive else return False
  58. if not self.downloading():
  59. self.running = False
  60. else:
  61. sleep(0.1)
  62. # If we reach here close down all child threads
  63. self.terminate_all()
  64. CallAfter(Publisher.sendMessage, PUBLISHER_TOPIC, DataPack('finish'))
  65. def downloading(self):
  66. for proc in self.procList:
  67. if proc.isAlive():
  68. return True
  69. return False
  70. def _add_download_item(self, downloadItem):
  71. self.downloadlist.append(downloadItem)
  72. def extract_data(self):
  73. data = self.downloadlist.pop(0)
  74. url = data['url']
  75. index = data['index']
  76. return url, index
  77. def terminate_all(self):
  78. for proc in self.procList:
  79. if proc.isAlive():
  80. proc.close()
  81. proc.join()
  82. def check_queue(self):
  83. for proc in self.procList:
  84. if not self.running: break
  85. if not proc.isAlive():
  86. return proc
  87. return None
  88. def close(self):
  89. self.procNo = 0
  90. self.running = False
  91. CallAfter(Publisher.sendMessage, PUBLISHER_TOPIC, DataPack('close'))
  92. class ProcessWrapper(Thread):
  93. def __init__(self, options, url, index, clear_dash_files, log=None):
  94. super(ProcessWrapper, self).__init__()
  95. self.clear_dash_files = clear_dash_files
  96. self.options = options
  97. self.index = index
  98. self.log = log
  99. self.url = url
  100. self.filenames = []
  101. self.stopped = False
  102. self.error = False
  103. self.proc = None
  104. self.start()
  105. def run(self):
  106. self.proc = self.create_process(self.get_cmd(), self.get_process_info())
  107. # while subprocess is alive and NOT the current thread
  108. while self.proc_is_alive():
  109. # read stdout, stderr from proc
  110. stdout, stderr = self.read()
  111. if stdout != '':
  112. # pass stdout to output formatter
  113. data = OutputFormatter(stdout).get_data()
  114. if self.clear_dash_files: self.add_file(data)
  115. # add index to data pack
  116. data.index = self.index
  117. # send data back to caller
  118. CallAfter(Publisher.sendMessage, PUBLISHER_TOPIC, data)
  119. if stderr != '':
  120. self.error = True
  121. self.write_to_log(stderr)
  122. if not self.stopped:
  123. if self.clear_dash_files:
  124. self.clear_dash()
  125. if not self.error:
  126. CallAfter(Publisher.sendMessage, PUBLISHER_TOPIC, DataPack('finish', self.index))
  127. else:
  128. CallAfter(Publisher.sendMessage, PUBLISHER_TOPIC, DataPack('error', self.index))
  129. def add_file(self, dataPack):
  130. if dataPack.header == 'filename':
  131. self.filenames.append(dataPack.data)
  132. def write_to_log(self, data):
  133. if self.log != None:
  134. self.log.write(data)
  135. def clear_dash(self):
  136. CallAfter(Publisher.sendMessage, PUBLISHER_TOPIC, DataPack('remove', self.index))
  137. for f in self.filenames:
  138. if file_exist(f):
  139. remove_file(f)
  140. def close(self):
  141. self.proc.kill()
  142. self.stopped = True
  143. CallAfter(Publisher.sendMessage, PUBLISHER_TOPIC, DataPack('close', self.index))
  144. def proc_is_alive(self):
  145. return self.proc.poll() == None
  146. def read(self):
  147. stdout = ''
  148. stderr = ''
  149. stdout = self.proc.stdout.readline()
  150. if stdout == '':
  151. stderr = self.proc.stderr.readline()
  152. return stdout.rstrip(), stderr.rstrip()
  153. def create_process(self, cmd, info):
  154. return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=info)
  155. def get_cmd(self):
  156. enc = get_encoding()
  157. if enc != None:
  158. cmd = encode_list(self.options + [self.url], enc)
  159. else:
  160. cmd = self.options + [self.url]
  161. return cmd
  162. def get_process_info(self):
  163. if get_os_type() == 'nt':
  164. info = subprocess.STARTUPINFO()
  165. info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
  166. return info
  167. else:
  168. return None