From d2c07a8827897b46729afe0ffc18aaa39b6dc3ac Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Thu, 16 Jan 2020 20:27:48 +0100 Subject: [PATCH 01/22] First variant of wexpect2, which is an upside down wexpect --- wexpect/console_reader.py | 438 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 438 insertions(+) create mode 100644 wexpect/console_reader.py diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py new file mode 100644 index 0000000..4d270a4 --- /dev/null +++ b/wexpect/console_reader.py @@ -0,0 +1,438 @@ +import time +import sys +import logging +import os +import re +import shutil +import types +import traceback +import signal +import pkg_resources +from io import StringIO + +from ctypes import windll +import pywintypes +from win32com.shell.shell import SHGetSpecialFolderPath +import win32console +import win32process +import win32con +import win32gui +import win32api +import win32file +import winerror +import win32pipe + +__version__ = 'test' + +# +# System-wide constants +# +screenbufferfillchar = '\4' +maxconsoleY = 8000 + +# The version is handled by the package: pbr, which derives the version from the git tags. +try: + __version__ = pkg_resources.require("wexpect")[0].version +except: # pragma: no cover + __version__ = '0.0.1.unkowndev0' + +# +# Create logger: We write logs only to file. Printing out logs are dangerous, because of the deep +# console manipulation. +# +logger = logging.getLogger('wexpect') +os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' +try: + logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] + logger.setLevel(logger_level) + fh = logging.FileHandler('wexpect.log', 'w', 'utf-8') + formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) +except KeyError: + logger.setLevel(logging.ERROR) + +# Test the logger +#logger.info('wexpect imported; logger working') + + +class ConsoleReader: + """Consol class (aka. client-side python class) for the child. + + This class initialize the console starts the child in it and reads the console periodically. + """ + + def __init__(self, path, parent_pid, parent_tid, cp=None): + """Initialize the console starts the child in it and reads the console periodically. + + + + Args: + path (str): Child's executable with arguments. + parent_pid (int): Parent (aka. host) process process-ID + parent_tid (int): Parent (aka. host) process thread-ID + cp (:obj:, optional): Output console code page. + """ + self.lastRead = 0 + self.__bufferY = 0 + self.lastReadData = "" + self.totalRead = 0 + self.__buffer = StringIO() + self.__currentReadCo = win32console.PyCOORDType(0, 0) + + + logger.info("ConsoleReader started") + logger.info("parent_tid %s" % parent_tid) + self.create_pipe() + if cp: + try: + logger.info("Setting console output code page to %s" % cp) + win32console.SetConsoleOutputCP(cp) + logger.info("Console output code page: %s" % windll.kernel32.GetConsoleOutputCP()) + except Exception as e: + logger.info(e) + + try: + logger.info('Spawning %s' % path) + try: + self.initConsole() + + time.sleep(1) + si = win32process.GetStartupInfo() + self.__childProcess, _, childPid, self.__tid = win32process.CreateProcess(None, path, None, None, False, + 0, None, None, si) + + print('123') + print('456') + print('789') + + except Exception as e: + logger.info(e) + time.sleep(.1) + return + + time.sleep(.1) + + paused = False + + + while True: + consinfo = self.consout.GetConsoleScreenBufferInfo() + cursorPos = consinfo['CursorPosition'] + + if win32process.GetExitCodeProcess(self.__childProcess) != win32con.STILL_ACTIVE: + time.sleep(.1) + try: + win32process.TerminateProcess(self.__childProcess, 0) + except pywintypes.error as e: + """ 'Access denied' happens always? Perhaps if not running as admin (or UAC + enabled under Vista/7). Don't log. Child process will exit regardless when + calling sys.exit + """ + if e.args[0] != winerror.ERROR_ACCESS_DENIED: + logger.info(e) + return + + if cursorPos.Y > maxconsoleY and not paused: + logger.info('cursorPos %s' % cursorPos) + self.suspendThread() + paused = True + + if cursorPos.Y <= maxconsoleY and paused: + logger.info('cursorPos %s' % cursorPos) + self.resumeThread() + paused = False + + time.sleep(.1) + except Exception as e: + logger.error(e) + time.sleep(.1) + + def create_pipe(self): + pid = win32process.GetCurrentProcessId() + pipe_name = 'wexpect_pipe_c2s_{}'.format(pid) + pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) + logger.info('Start pipe server: %s', pipe_full_path) + self.pipe = win32pipe.CreateNamedPipe( + pipe_full_path, + win32pipe.PIPE_ACCESS_DUPLEX, + win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT, + 1, 65536, 65536, 0, None) + logger.info("waiting for client") + win32pipe.ConnectNamedPipe(self.pipe, None) + logger.info('got client') + + def write_pipe(self, msg): + # convert to bytes + msg_bytes = str.encode(msg) + win32file.WriteFile(self.pipe, msg_bytes) + + def initConsole(self, consout=None): + if not consout: + consout=self.getConsoleOut() + + rect = win32console.PySMALL_RECTType(0, 0, 79, 24) + consout.SetConsoleWindowInfo(True, rect) + size = win32console.PyCOORDType(80, 16000) + consout.SetConsoleScreenBufferSize(size) + pos = win32console.PyCOORDType(0, 0) + # Use NUL as fill char because it displays as whitespace + # (if we interact() with the child) + consout.FillConsoleOutputCharacter(screenbufferfillchar, size.X * size.Y, pos) + + consinfo = consout.GetConsoleScreenBufferInfo() + self.__consSize = consinfo['Size'] + logger.info('self.__consSize: ' + str(self.__consSize)) + self.startCursorPos = consinfo['CursorPosition'] + + + def parseData(self, s): + """Ensures that special characters are interpretted as + newlines or blanks, depending on if there written over + characters or screen-buffer-fill characters.""" + + strlist = [] + for i, c in enumerate(s): + if c == screenbufferfillchar: + if (self.totalRead - self.lastRead + i + 1) % self.__consSize.X == 0: + strlist.append('\r\n') + else: + strlist.append(c) + + s = ''.join(strlist) + return s + + def getConsoleOut(self): + consfile = win32file.CreateFile('CONOUT$', + win32con.GENERIC_READ | win32con.GENERIC_WRITE, + win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE, + None, + win32con.OPEN_EXISTING, + 0, + 0) + + self.consout = win32console.PyConsoleScreenBufferType(consfile) + return self.consout + + def getCoord(self, offset): + """Converts an offset to a point represented as a tuple.""" + + x = offset % self.__consSize.X + y = offset // self.__consSize.X + return win32console.PyCOORDType(x, y) + + def getOffset(self, coord): + """Converts a tuple-point to an offset.""" + + return coord.X + coord.Y * self.__consSize.X + + def readConsole(self, startCo, endCo): + """Reads the console area from startCo to endCo and returns it + as a string.""" + + if startCo is None: + startCo = self.startCursorPos + startCo.Y = startCo.Y + + if endCo is None: + consinfo = self.consout.GetConsoleScreenBufferInfo() + endCo = consinfo['CursorPosition'] + endCo= self.getCoord(0 + self.getOffset(endCo)) + # endCo.Y = endCo.Y+1 + # logger.info(endCo.Y+1) + + logger.info(startCo) + logger.info(endCo) + + buff = [] + self.lastRead = 0 + + while True: + startOff = self.getOffset(startCo) + endOff = self.getOffset(endCo) + readlen = endOff - startOff + + if readlen <= 0: + break + + if readlen > 4000: + readlen = 4000 + endPoint = self.getCoord(startOff + readlen) + + s = self.consout.ReadConsoleOutputCharacter(readlen, startCo) + self.lastRead += len(s) + self.totalRead += len(s) + buff.append(s) + + startCo = endPoint + + logger.info(repr(s)) + return ''.join(buff) + + + def readConsoleToCursor(self): + """Reads from the current read position to the current cursor + position and inserts the string into self.__buffer.""" + + if not self.consout: + return "" + + consinfo = self.consout.GetConsoleScreenBufferInfo() + cursorPos = consinfo['CursorPosition'] + + logger.debug('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) + + isSameX = cursorPos.X == self.__currentReadCo.X + isSameY = cursorPos.Y == self.__currentReadCo.Y + isSamePos = isSameX and isSameY + + logger.debug('isSameY: %r' % isSameY) + logger.debug('isSamePos: %r' % isSamePos) + + if isSameY or not self.lastReadData.endswith('\r\n'): + # Read the current slice again + self.totalRead -= self.lastRead + self.__currentReadCo.X = 0 + self.__currentReadCo.Y = self.__bufferY + + logger.debug('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) + + raw = self.readConsole(self.__currentReadCo, cursorPos) + rawlist = [] + while raw: + rawlist.append(raw[:self.__consSize.X]) + raw = raw[self.__consSize.X:] + raw = ''.join(rawlist) + s = self.parseData(raw) + logger.debug(s) + for i, line in enumerate(reversed(rawlist)): + if line.endswith(screenbufferfillchar): + # Record the Y offset where the most recent line break was detected + self.__bufferY += len(rawlist) - i + break + + logger.debug('lastReadData: %r' % self.lastReadData) + logger.debug('s: %r' % s) + + if isSamePos and self.lastReadData == s: + logger.debug('isSamePos and self.lastReadData == s') + s = '' + + logger.debug('s: %r' % s) + + if s: + lastReadData = self.lastReadData + pos = self.getOffset(self.__currentReadCo) + self.lastReadData = s + if isSameY or not lastReadData.endswith('\r\n'): + # Detect changed lines + self.__buffer.seek(pos) + buf = self.__buffer.read() + logger.debug('buf: %r' % buf) + logger.debug('raw: %r' % raw) + if raw.startswith(buf): + # Line has grown + rawslice = raw[len(buf):] + # Update last read bytes so line breaks can be detected in parseData + lastRead = self.lastRead + self.lastRead = len(rawslice) + s = self.parseData(rawslice) + self.lastRead = lastRead + else: + # Cursor has been repositioned + s = '\r' + s + logger.debug('s: %r' % s) + self.__buffer.seek(pos) + self.__buffer.truncate() + self.__buffer.write(raw) + + self.__currentReadCo.X = cursorPos.X + self.__currentReadCo.Y = cursorPos.Y + + return s + + +def client(path, pid, tid): + try: + w = ConsoleReader(path, pid, tid) + time.sleep(1) + w.write_pipe(w.readConsoleToCursor()) + except Exception: + tb = traceback.format_exc() + logger.error(tb) + + +def pipe_client(conpid): + pipe_name = 'wexpect_pipe_c2s_{}'.format(conpid) + pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) + print('Trying to connect to pipe: {}'.format(pipe_full_path)) + quit = False + + while not quit: + try: + handle = win32file.CreateFile( + pipe_full_path, + win32file.GENERIC_READ | win32file.GENERIC_WRITE, + 0, + None, + win32file.OPEN_EXISTING, + 0, + None + ) + res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) + if res == 0: + print(f"SetNamedPipeHandleState return code: {res}") + while True: + resp = win32file.ReadFile(handle, 64*1024) + print(f"message: {resp}") + except pywintypes.error as e: + if e.args[0] == 2: + print("no pipe, trying again in a bit") + time.sleep(0.2) + elif e.args[0] == 109: + print("broken pipe, bye bye") + quit = True + + +def main(): + pass + + +if __name__ == '__main__': + + si = win32process.GetStartupInfo() + si.dwFlags = win32process.STARTF_USESHOWWINDOW + si.wShowWindow = win32con.SW_HIDE + pyargs = ['-c'] + + dirname = os.path.dirname(sys.executable + if getattr(sys, 'frozen', False) else + os.path.abspath(__file__)) + +# client('uname') + + pid = win32process.GetCurrentProcessId() + tid = win32api.GetCurrentThreadId() + + commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') + if getattr(sys, 'frozen', False) else + os.path.join(os.path.dirname(sys.executable), 'python.exe'), + ' '.join(pyargs), + "import sys;" + "sys.path.append('D:\\\\bt\\\\wexpect');" + "import console_reader;" + "import time;" + "console_reader.client('uname', {tid}, {pid});".format(pid=pid, tid=tid) + ) + + + print(commandLine) + + __oproc, _, conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, + win32process.CREATE_NEW_CONSOLE, None, None, si) +# time.sleep(3) + pipe_client(conpid) + + + time.sleep(5) + \ No newline at end of file From c7a4e092c858528d9ebd6388bc4089e26fa10646 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Fri, 17 Jan 2020 16:45:48 +0100 Subject: [PATCH 02/22] [ADD] Basic features added --- wexpect.py | 1940 ------------------------------------- wexpect/__init__.py | 16 + wexpect/console_reader.py | 330 ++++--- wexpect/spawn.py | 849 ++++++++++++++++ wexpect/wexpect_util.py | 105 ++ 5 files changed, 1156 insertions(+), 2084 deletions(-) delete mode 100644 wexpect.py create mode 100644 wexpect/__init__.py create mode 100644 wexpect/spawn.py create mode 100644 wexpect/wexpect_util.py diff --git a/wexpect.py b/wexpect.py deleted file mode 100644 index decaffa..0000000 --- a/wexpect.py +++ /dev/null @@ -1,1940 +0,0 @@ -"""Wexpect is a Windows variant of pexpect https://pexpect.readthedocs.io. - -Wexpect is a Python module for spawning child applications and controlling -them automatically. Wexpect can be used for automating interactive applications -such as ssh, ftp, passwd, telnet, etc. It can be used to a automate setup -scripts for duplicating software package installations on different servers. It -can be used for automated software testing. Wexpect is in the spirit of Don -Libes' Expect, but Wexpect is pure Python. Other Expect-like modules for Python -require TCL and Expect or require C extensions to be compiled. Wexpect does not -use C, Expect, or TCL extensions. - -There are two main interfaces to Wexpect -- the function, run() and the class, -spawn. You can call the run() function to execute a command and return the -output. This is a handy replacement for os.system(). - -For example:: - - wexpect.run('ls -la') - -The more powerful interface is the spawn class. You can use this to spawn an -external child command and then interact with the child by sending lines and -expecting responses. - -For example:: - - child = wexpect.spawn('scp foo myname@host.example.com:.') - child.expect('Password:') - child.sendline(mypassword) - -This works even for commands that ask for passwords or other input outside of -the normal stdio streams. - -Credits: Noah Spurrier, Richard Holden, Marco Molteni, Kimberley Burchett, -Robert Stone, Hartmut Goebel, Chad Schroeder, Erick Tryzelaar, Dave Kirby, Ids -vander Molen, George Todd, Noel Taylor, Nicolas D. Cesar, Alexander Gattin, -Geoffrey Marshall, Francisco Lourenco, Glen Mabey, Karthik Gurusamy, Fernando -Perez, Corey Minyard, Jon Cohen, Guillaume Chazarain, Andrew Ryan, Nick -Craig-Wood, Andrew Stone, Jorgen Grahn, Benedek Racz - -Free, open source, and all that good stuff. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -Wexpect Copyright (c) 2019 Benedek Racz - -""" - -# -# wexpect is windows only. Use pexpect on linux like systems. -# -import sys -if sys.platform != 'win32': # pragma: no cover - raise ImportError (str(e) + """ -sys.platform != 'win32': Wexpect supports only Windows. -Pexpect is intended for UNIX-like operating systems.""") - -# -# Import built in modules -# -import warnings -import logging -import os -import time -import re -import select -import shutil -import struct -import types -import errno -import traceback -import signal -import pkg_resources -from io import StringIO - -try: - from ctypes import windll - import pywintypes - from win32com.shell.shellcon import CSIDL_APPDATA - from win32com.shell.shell import SHGetSpecialFolderPath - import win32console - import win32process - import win32con - import win32gui - import win32api - import win32file - import winerror -except ImportError as e: # pragma: no cover - raise ImportError(str(e) + "\nThis package requires the win32 python packages.\r\nInstall with pip install pywin32") - -# -# System-wide constants -# -screenbufferfillchar = '\4' -maxconsoleY = 8000 - -# The version is handled by the package: pbr, which derives the version from the git tags. -try: - __version__ = pkg_resources.require("wexpect")[0].version -except: # pragma: no cover - __version__ = '0.0.1.unkowndev0' - -__all__ = ['ExceptionPexpect', 'EOF', 'TIMEOUT', 'spawn', 'run', 'which', - 'split_command_line', '__version__'] - -# -# Create logger: We write logs only to file. Printing out logs are dangerous, because of the deep -# console manipulation. -# -logger = logging.getLogger('wexpect') -try: - logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] - logger.setLevel(logger_level) - fh = logging.FileHandler('wexpect.log', 'w', 'utf-8') - formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') - fh.setFormatter(formatter) - logger.addHandler(fh) -except KeyError as _: - logger.setLevel(logging.ERROR) - -# Test the logger -logger.info('wexpect imported; logger working') - -#################################################################################################### -# -# Exceptions -# -#################################################################################################### - -class ExceptionPexpect(Exception): - """Base class for all exceptions raised by this module. - """ - - def __init__(self, value): - - self.value = value - - def __str__(self): - - return str(self.value) - - def get_trace(self): - """This returns an abbreviated stack trace with lines that only concern - the caller. In other words, the stack trace inside the Wexpect module - is not included. """ - - tblist = traceback.extract_tb(sys.exc_info()[2]) - tblist = [item for item in tblist if self.__filter_not_wexpect(item)] - tblist = traceback.format_list(tblist) - return ''.join(tblist) - - def __filter_not_wexpect(self, trace_list_item): - """This returns True if list item 0 the string 'wexpect.py' in it. """ - - if trace_list_item[0].find('wexpect.py') == -1: - return True - else: - return False - - -class EOF(ExceptionPexpect): - """Raised when EOF is read from a child. This usually means the child has exited. - The user can wait to EOF, which means he waits the end of the execution of the child process.""" - -class TIMEOUT(ExceptionPexpect): - """Raised when a read time exceeds the timeout. """ - - -def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None, logfile=None, - cwd=None, env=None, echo=True): - - """ - This function runs the given command; waits for it to finish; then - returns all output as a string. STDERR is included in output. If the full - path to the command is not given then the path is searched. - - Note that lines are terminated by CR/LF (\\r\\n) combination even on - UNIX-like systems because this is the standard for pseudo ttys. If you set - 'withexitstatus' to true, then run will return a tuple of (command_output, - exitstatus). If 'withexitstatus' is false then this returns just - command_output. - - The run() function can often be used instead of creating a spawn instance. - For example, the following code uses spawn:: - - child = spawn('scp foo myname@host.example.com:.') - child.expect ('(?i)password') - child.sendline (mypassword) - - The previous code can be replace with the following:: - - Examples - ======== - - Start the apache daemon on the local machine:: - - run ("/usr/local/apache/bin/apachectl start") - - Check in a file using SVN:: - - run ("svn ci -m 'automatic commit' my_file.py") - - Run a command and capture exit status:: - - (command_output, exitstatus) = run ('ls -l /bin', withexitstatus=1) - - Tricky Examples - =============== - - The following will run SSH and execute 'ls -l' on the remote machine. The - password 'secret' will be sent if the '(?i)password' pattern is ever seen:: - - run ("ssh username@machine.example.com 'ls -l'", events={'(?i)password':'secret\\n'}) - - The 'events' argument should be a dictionary of patterns and responses. - Whenever one of the patterns is seen in the command out run() will send the - associated response string. Note that you should put newlines in your - string if Enter is necessary. The responses may also contain callback - functions. Any callback is function that takes a dictionary as an argument. - The dictionary contains all the locals from the run() function, so you can - access the child spawn object or any other variable defined in run() - (event_count, child, and extra_args are the most useful). A callback may - return True to stop the current run process otherwise run() continues until - the next event. A callback may also return a string which will be sent to - the child. 'extra_args' is not used by directly run(). It provides a way to - pass data to a callback function through run() through the locals - dictionary passed to a callback. """ - - if timeout == -1: - child = spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env) - else: - child = spawn(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env) - if events is not None: - patterns = list(events.keys()) - responses = list(events.values()) - else: - patterns=None # We assume that EOF or TIMEOUT will save us. - responses=None - child_result_list = [] - event_count = 0 - while 1: - try: - index = child.expect (patterns) - if type(child.after) in (str,): - child_result_list.append(child.before + child.after) - else: # child.after may have been a TIMEOUT or EOF, so don't cat those. - child_result_list.append(child.before) - if type(responses[index]) in (str,): - child.send(responses[index]) - elif type(responses[index]) is types.FunctionType: - callback_result = responses[index](locals()) - sys.stdout.flush() - if type(callback_result) in (str,): - child.send(callback_result) - elif callback_result: - break - else: - raise TypeError ('The callback must be a string or function type.') - event_count = event_count + 1 - except TIMEOUT as e: - child_result_list.append(child.before) - break - except EOF as e: - child_result_list.append(child.before) - break - child_result = ''.join(child_result_list) - if withexitstatus: - child.close() - return (child_result, child.exitstatus) - else: - return child_result - -def spawn(command, args=[], timeout=30, maxread=2000, searchwindowsize=None, logfile=None, cwd=None, - env=None, codepage=None, echo=True): - """This is the most essential function. The command parameter may be a string that - includes a command and any arguments to the command. For example:: - - child = wexpect.spawn ('/usr/bin/ftp') - child = wexpect.spawn ('/usr/bin/ssh user@example.com') - child = wexpect.spawn ('ls -latr /tmp') - - You may also construct it with a list of arguments like so:: - - child = wexpect.spawn ('/usr/bin/ftp', []) - child = wexpect.spawn ('/usr/bin/ssh', ['user@example.com']) - child = wexpect.spawn ('ls', ['-latr', '/tmp']) - - After this the child application will be created and will be ready to - talk to. For normal use, see expect() and send() and sendline(). - - Remember that Wexpect does NOT interpret shell meta characters such as - redirect, pipe, or wild cards (>, |, or *). This is a common mistake. - If you want to run a command and pipe it through another command then - you must also start a shell. For example:: - - child = wexpect.spawn('/bin/bash -c "ls -l | grep LOG > log_list.txt"') - child.expect(wexpect.EOF) - - The second form of spawn (where you pass a list of arguments) is useful - in situations where you wish to spawn a command and pass it its own - argument list. This can make syntax more clear. For example, the - following is equivalent to the previous example:: - - shell_cmd = 'ls -l | grep LOG > log_list.txt' - child = wexpect.spawn('/bin/bash', ['-c', shell_cmd]) - child.expect(wexpect.EOF) - - The maxread attribute sets the read buffer size. This is maximum number - of bytes that Wexpect will try to read from a TTY at one time. Setting - the maxread size to 1 will turn off buffering. Setting the maxread - value higher may help performance in cases where large amounts of - output are read back from the child. This feature is useful in - conjunction with searchwindowsize. - - The searchwindowsize attribute sets the how far back in the incomming - seach buffer Wexpect will search for pattern matches. Every time - Wexpect reads some data from the child it will append the data to the - incomming buffer. The default is to search from the beginning of the - imcomming buffer each time new data is read from the child. But this is - very inefficient if you are running a command that generates a large - amount of data where you want to match The searchwindowsize does not - effect the size of the incomming data buffer. You will still have - access to the full buffer after expect() returns. - - The delaybeforesend helps overcome a weird behavior that many users - were experiencing. The typical problem was that a user would expect() a - "Password:" prompt and then immediately call sendline() to send the - password. The user would then see that their password was echoed back - to them. Passwords don't normally echo. The problem is caused by the - fact that most applications print out the "Password" prompt and then - turn off stdin echo, but if you send your password before the - application turned off echo, then you get your password echoed. - Normally this wouldn't be a problem when interacting with a human at a - real keyboard. If you introduce a slight delay just before writing then - this seems to clear up the problem. This was such a common problem for - many users that I decided that the default wexpect behavior should be - to sleep just before writing to the child application. 1/20th of a - second (50 ms) seems to be enough to clear up the problem. You can set - delaybeforesend to 0 to return to the old behavior. Most Linux machines - don't like this to be below 0.03. I don't know why. - - Note that spawn is clever about finding commands on your path. - It uses the same logic that "which" uses to find executables. - - If you wish to get the exit status of the child you must call the - close() method. The exit status of the child will be stored in self.exitstatus. - If the child exited normally then exitstatus will store the exit return code. - """ - - logger.debug('=' * 80) - logger.debug('Buffer size: %s' % maxread) - if searchwindowsize: - logger.debug('Search window size: %s' % searchwindowsize) - logger.debug('Timeout: %ss' % timeout) - if env: - logger.debug('Environment:') - for name in env: - logger.debug('\t%s=%s' % (name, env[name])) - if cwd: - logger.debug('Working directory: %s' % cwd) - - return spawn_windows(command, args, timeout, maxread, searchwindowsize, logfile, cwd, env, - codepage, echo=echo) - -class spawn_windows (): - """This is the main class interface for Wexpect. Use this class to start - and control child applications. """ - - def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, - logfile=None, cwd=None, env=None, codepage=None, echo=True): - """ The spawn_windows constructor. Do not call it directly. Use spawn(), or run() instead. - """ - self.codepage = codepage - - self.stdin = sys.stdin - self.stdout = sys.stdout - self.stderr = sys.stderr - - self.searcher = None - self.ignorecase = False - self.before = None - self.after = None - self.match = None - self.match_index = None - self.terminated = True - self.exitstatus = None - self.status = None # status returned by os.waitpid - self.flag_eof = False - self.flag_child_finished = False - self.pid = None - self.child_fd = -1 # initially closed - self.timeout = timeout - self.delimiter = EOF - self.logfile = logfile - self.logfile_read = None # input from child (read_nonblocking) - self.logfile_send = None # output to send (send, sendline) - self.maxread = maxread # max bytes to read at one time into buffer - self.buffer = '' # This is the read buffer. See maxread. - self.searchwindowsize = searchwindowsize # Anything before searchwindowsize point is preserved, but not searched. - self.delaybeforesend = 0.05 # Sets sleep time used just before sending data to child. Time in seconds. - self.delayafterterminate = 0.1 # Sets delay in terminate() method to allow kernel time to update process status. Time in seconds. - self.name = '<' + repr(self) + '>' # File-like object. - self.closed = True # File-like object. - self.ocwd = os.getcwd() - self.cwd = cwd - self.env = env - - # allow dummy instances for subclasses that may not use command or args. - if command is None: - self.command = None - self.args = None - self.name = '' - else: - self._spawn(command, args, echo=echo) - - def __del__(self): - """This makes sure that no system resources are left open. Python only - garbage collects Python objects, not the child console.""" - - try: - self.wtty.terminate_child() - except: - pass - - def __str__(self): - - """This returns a human-readable string that represents the state of - the object. """ - - s = [] - s.append(repr(self)) - s.append('command: ' + str(self.command)) - s.append('args: ' + str(self.args)) - s.append('searcher: ' + str(self.searcher)) - s.append('buffer (last 100 chars): ' + str(self.buffer)[-100:]) - s.append('before (last 100 chars): ' + str(self.before)[-100:]) - s.append('after: ' + str(self.after)) - s.append('match: ' + str(self.match)) - s.append('match_index: ' + str(self.match_index)) - s.append('exitstatus: ' + str(self.exitstatus)) - s.append('flag_eof: ' + str(self.flag_eof)) - s.append('pid: ' + str(self.pid)) - s.append('child_fd: ' + str(self.child_fd)) - s.append('closed: ' + str(self.closed)) - s.append('timeout: ' + str(self.timeout)) - s.append('delimiter: ' + str(self.delimiter)) - s.append('logfile: ' + str(self.logfile)) - s.append('logfile_read: ' + str(self.logfile_read)) - s.append('logfile_send: ' + str(self.logfile_send)) - s.append('maxread: ' + str(self.maxread)) - s.append('ignorecase: ' + str(self.ignorecase)) - s.append('searchwindowsize: ' + str(self.searchwindowsize)) - s.append('delaybeforesend: ' + str(self.delaybeforesend)) - s.append('delayafterterminate: ' + str(self.delayafterterminate)) - return '\n'.join(s) - - def _spawn(self,command,args=[], echo=True): - """This starts the given command in a child process. This does all the - fork/exec type of stuff for a pty. This is called by __init__. If args - is empty then command will be parsed (split on spaces) and args will be - set to parsed arguments. """ - - # The pid and child_fd of this object get set by this method. - # Note that it is difficult for this method to fail. - # You cannot detect if the child process cannot start. - # So the only way you can tell if the child process started - # or not is to try to read from the file descriptor. If you get - # EOF immediately then it means that the child is already dead. - # That may not necessarily be bad because you may haved spawned a child - # that performs some task; creates no stdout output; and then dies. - - # If command is an int type then it may represent a file descriptor. - if type(command) == type(0): - raise ExceptionPexpect ('Command is an int type. If this is a file descriptor then maybe you want to use fdpexpect.fdspawn which takes an existing file descriptor instead of a command string.') - - if type (args) != type([]): - raise TypeError ('The argument, args, must be a list.') - - if args == []: - self.args = split_command_line(command) - self.command = self.args[0] - else: - self.args = args[:] # work with a copy - self.args.insert (0, command) - self.command = command - - command_with_path = shutil.which(self.command) - if command_with_path is None: - raise ExceptionPexpect ('The command was not found or was not executable: %s.' % self.command) - self.command = command_with_path - self.args[0] = self.command - - self.name = '<' + ' '.join (self.args) + '>' - - #assert self.pid is None, 'The pid member should be None.' - #assert self.command is not None, 'The command member should not be None.' - - self.wtty = Wtty(codepage=self.codepage, echo=echo) - - if self.cwd is not None: - os.chdir(self.cwd) - - self.child_fd = self.wtty.spawn(self.command, self.args, self.env) - - if self.cwd is not None: - # Restore the original working dir - os.chdir(self.ocwd) - - self.terminated = False - self.closed = False - self.pid = self.wtty.pid - - - def fileno (self): # File-like object. - """There is no child fd.""" - - return 0 - - def close(self, force=True): # File-like object. - """ Closes the child console.""" - - self.closed = self.terminate(force) - if not self.closed: - raise ExceptionPexpect ('close() could not terminate the child using terminate()') - self.closed = True - - def isatty(self): # File-like object. - """The child is always created with a console.""" - - return True - - def waitnoecho (self, timeout=-1): - """This waits until the terminal ECHO flag is set False. This returns - True if the echo mode is off. This returns False if the ECHO flag was - not set False before the timeout. This can be used to detect when the - child is waiting for a password. Usually a child application will turn - off echo mode when it is waiting for the user to enter a password. For - example, instead of expecting the "password:" prompt you can wait for - the child to set ECHO off:: - - p = wexpect.spawn ('ssh user@example.com') - p.waitnoecho() - p.sendline(mypassword) - - If timeout is None then this method to block forever until ECHO flag is - False. - - """ - - if timeout == -1: - timeout = self.timeout - if timeout is not None: - end_time = time.time() + timeout - while True: - if not self.getecho(): - return True - if timeout < 0 and timeout is not None: - return False - if timeout is not None: - timeout = end_time - time.time() - time.sleep(0.1) - - def getecho (self): - """This returns the terminal echo mode. This returns True if echo is - on or False if echo is off. Child applications that are expecting you - to enter a password often set ECHO False. See waitnoecho().""" - - return self.wtty.getecho() - - def setecho (self, state): - """This sets the terminal echo mode on or off.""" - - self.wtty.setecho(state) - - def read (self, size = -1): # File-like object. - - """This reads at most "size" bytes from the file (less if the read hits - EOF before obtaining size bytes). If the size argument is negative or - omitted, read all data until EOF is reached. The bytes are returned as - a string object. An empty string is returned when EOF is encountered - immediately. """ - - if size == 0: - return '' - if size < 0: - self.expect (self.delimiter) # delimiter default is EOF - return self.before - - # I could have done this more directly by not using expect(), but - # I deliberately decided to couple read() to expect() so that - # I would catch any bugs early and ensure consistant behavior. - # It's a little less efficient, but there is less for me to - # worry about if I have to later modify read() or expect(). - # Note, it's OK if size==-1 in the regex. That just means it - # will never match anything in which case we stop only on EOF. - cre = re.compile('.{%d}' % size, re.DOTALL) - index = self.expect ([cre, self.delimiter]) # delimiter default is EOF - if index == 0: - return self.after ### self.before should be ''. Should I assert this? - return self.before - - def readline (self, size = -1): # File-like object. - - """This reads and returns one entire line. A trailing newline is kept - in the string, but may be absent when a file ends with an incomplete - line. Note: This readline() looks for a \\r\\n pair even on UNIX - because this is what the pseudo tty device returns. So contrary to what - you may expect you will receive the newline as \\r\\n. An empty string - is returned when EOF is hit immediately. Currently, the size argument is - mostly ignored, so this behavior is not standard for a file-like - object. If size is 0 then an empty string is returned. """ - - if size == 0: - return '' - index = self.expect (['\r\n', self.delimiter]) # delimiter default is EOF - if index == 0: - return self.before + '\r\n' - else: - return self.before - - def __iter__ (self): # File-like object. - - """This is to support iterators over a file-like object. - """ - - return self - - def __next__ (self): # File-like object. - - """This is to support iterators over a file-like object. - """ - - result = self.readline() - if self.after == self.delimiter: - raise StopIteration - return result - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.terminate() - - def readlines (self, sizehint = -1): # File-like object. - - """This reads until EOF using readline() and returns a list containing - the lines thus read. The optional "sizehint" argument is ignored. """ - - lines = [] - while True: - line = self.readline() - if not line: - break - lines.append(line) - return lines - - def read_nonblocking (self, size = 1): - """This reads at most size characters from the child application. If - the end of file is read then an EOF exception will be raised. - - This is not effected by the 'size' parameter, so if you call - read_nonblocking(size=100, timeout=30) and only one character is - available right away then one character will be returned immediately. - It will not wait for 30 seconds for another 99 characters to come in. - - This is a wrapper around Wtty.read(). """ - - if self.closed: - raise ValueError ('I/O operation on closed file in read_nonblocking().') - - try: - # The real child and it's console are two different process. The console dies 0.1 sec - # later to be able to read the child's last output (before EOF). So here we check - # isalive() (which checks the real child.) and try a last read on the console. To catch - # the last output. - # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. - if self.flag_child_finished: - raise EOF('self.self.flag_child_finished') - if not self.isalive(): - self.flag_child_finished = True - s = self.wtty.read_nonblocking(size) - except EOF: - self.flag_eof = True - raise - - if self.logfile is not None: - self.logfile.write (s) - self.logfile.flush() - if self.logfile_read is not None: - self.logfile_read.write (s) - self.logfile_read.flush() - - return s - - def write(self, s): # File-like object. - - """This is similar to send() except that there is no return value. - """ - - self.send (s) - - def writelines (self, sequence): # File-like object. - - """This calls write() for each element in the sequence. The sequence - can be any iterable object producing strings, typically a list of - strings. This does not add line separators There is no return value. - """ - - for s in sequence: - self.write (s) - - def sendline(self, s=''): - - """This is like send(), but it adds a line feed (os.linesep). This - returns the number of bytes written. """ - - n = self.send(s) - n = n + self.send (os.linesep) - return n - - def sendeof(self): - - """This sends an EOF to the child. This sends a character which causes - the pending parent output buffer to be sent to the waiting child - program without waiting for end-of-line. If it is the first character - of the line, the read() in the user program returns 0, which signifies - end-of-file. This means to work as expected a sendeof() has to be - called at the beginning of a line. This method does not send a newline. - It is the responsibility of the caller to ensure the eof is sent at the - beginning of a line. """ - - # platform does not define VEOF so assume CTRL-D - char = chr(4) - self.send(char) - - def send(self, s): - """This sends a string to the child process. This returns the number of - bytes written. If a log file was set then the data is also written to - the log. """ - - (self.delaybeforesend) - if self.logfile is not None: - self.logfile.write (s) - self.logfile.flush() - if self.logfile_send is not None: - self.logfile_send.write (s) - self.logfile_send.flush() - c = self.wtty.write(s) - return c - - def sendintr(self): - """This sends a SIGINT to the child. It does not require - the SIGINT to be the first character on a line. """ - - self.wtty.sendintr() - - def eof (self): - - """This returns True if the EOF exception was ever raised. - """ - - return self.flag_eof - - def terminate(self, force=False): - """Terminate the child. Force not used. """ - - if not self.isalive(): - return True - - self.wtty.terminate_child() - time.sleep(self.delayafterterminate) - if not self.isalive(): - return True - - return False - - def kill(self, sig): - """Sig == sigint for ctrl-c otherwise the child is terminated.""" - - if not self.isalive(): - return - - if sig == signal.SIGINT: - self.wtty.sendintr() - else: - self.wtty.terminate_child() - - def wait(self): - """This waits until the child exits. This is a blocking call. This will - not read any data from the child, so this will block forever if the - child has unread output and has terminated. In other words, the child - may have printed output then called exit(); but, technically, the child - is still alive until its output is read.""" - - # We can't use os.waitpid under Windows because of 'permission denied' - # exception? Perhaps if not running as admin (or UAC enabled under - # Vista/7). Simply loop and wait for child to exit. - while self.isalive(): - time.sleep(.05) # Keep CPU utilization down - - return self.exitstatus - - def isalive(self): - """Determines if the child is still alive.""" - - if self.terminated: - return False - - if self.wtty.isalive(): - return True - else: - self.exitstatus = win32process.GetExitCodeProcess(self.wtty.getchild()) - self.status = (self.pid, self.exitstatus << 8) # left-shift exit status by 8 bits like os.waitpid - self.terminated = True - return False - - def compile_pattern_list(self, patterns): - - """This compiles a pattern-string or a list of pattern-strings. - Patterns must be a StringType, EOF, TIMEOUT, SRE_Pattern, or a list of - those. Patterns may also be None which results in an empty list (you - might do this if waiting for an EOF or TIMEOUT condition without - expecting any pattern). - - This is used by expect() when calling expect_list(). Thus expect() is - nothing more than:: - - cpl = self.compile_pattern_list(pl) - return self.expect_list(cpl, timeout) - - If you are using expect() within a loop it may be more - efficient to compile the patterns first and then call expect_list(). - This avoid calls in a loop to compile_pattern_list():: - - cpl = self.compile_pattern_list(my_pattern) - while some_condition: - ... - i = self.expect_list(clp, timeout) - ... - """ - - if patterns is None: - return [] - if type(patterns) is not list: - patterns = [patterns] - - compile_flags = re.DOTALL # Allow dot to match \n - if self.ignorecase: - compile_flags = compile_flags | re.IGNORECASE - compiled_pattern_list = [] - for p in patterns: - if type(p) in (str,): - compiled_pattern_list.append(re.compile(p, compile_flags)) - elif p is EOF: - compiled_pattern_list.append(EOF) - elif p is TIMEOUT: - compiled_pattern_list.append(TIMEOUT) - elif type(p) is type(re.compile('')): - compiled_pattern_list.append(p) - else: - raise TypeError ('Argument must be one of StringTypes, EOF, TIMEOUT, SRE_Pattern, or a list of those type. %s' % str(type(p))) - - return compiled_pattern_list - - def expect(self, pattern, timeout = -1, searchwindowsize=None): - - """This seeks through the stream until a pattern is matched. The - pattern is overloaded and may take several types. The pattern can be a - StringType, EOF, a compiled re, or a list of any of those types. - Strings will be compiled to re types. This returns the index into the - pattern list. If the pattern was not a list this returns index 0 on a - successful match. This may raise exceptions for EOF or TIMEOUT. To - avoid the EOF or TIMEOUT exceptions add EOF or TIMEOUT to the pattern - list. That will cause expect to match an EOF or TIMEOUT condition - instead of raising an exception. - - If you pass a list of patterns and more than one matches, the first match - in the stream is chosen. If more than one pattern matches at that point, - the leftmost in the pattern list is chosen. For example:: - - # the input is 'foobar' - index = p.expect (['bar', 'foo', 'foobar']) - # returns 1 ('foo') even though 'foobar' is a "better" match - - Please note, however, that buffering can affect this behavior, since - input arrives in unpredictable chunks. For example:: - - # the input is 'foobar' - index = p.expect (['foobar', 'foo']) - # returns 0 ('foobar') if all input is available at once, - # but returs 1 ('foo') if parts of the final 'bar' arrive late - - After a match is found the instance attributes 'before', 'after' and - 'match' will be set. You can see all the data read before the match in - 'before'. You can see the data that was matched in 'after'. The - re.MatchObject used in the re match will be in 'match'. If an error - occurred then 'before' will be set to all the data read so far and - 'after' and 'match' will be None. - - If timeout is -1 then timeout will be set to the self.timeout value. - - A list entry may be EOF or TIMEOUT instead of a string. This will - catch these exceptions and return the index of the list entry instead - of raising the exception. The attribute 'after' will be set to the - exception type. The attribute 'match' will be None. This allows you to - write code like this:: - - index = p.expect (['good', 'bad', wexpect.EOF, wexpect.TIMEOUT]) - if index == 0: - do_something() - elif index == 1: - do_something_else() - elif index == 2: - do_some_other_thing() - elif index == 3: - do_something_completely_different() - - instead of code like this:: - - try: - index = p.expect (['good', 'bad']) - if index == 0: - do_something() - elif index == 1: - do_something_else() - except EOF: - do_some_other_thing() - except TIMEOUT: - do_something_completely_different() - - These two forms are equivalent. It all depends on what you want. You - can also just expect the EOF if you are waiting for all output of a - child to finish. For example:: - - p = wexpect.spawn('/bin/ls') - p.expect (wexpect.EOF) - print p.before - - If you are trying to optimize for speed then see expect_list(). - """ - - compiled_pattern_list = self.compile_pattern_list(pattern) - return self.expect_list(compiled_pattern_list, timeout, searchwindowsize) - - def expect_list(self, pattern_list, timeout = -1, searchwindowsize = -1): - - """This takes a list of compiled regular expressions and returns the - index into the pattern_list that matched the child output. The list may - also contain EOF or TIMEOUT (which are not compiled regular - expressions). This method is similar to the expect() method except that - expect_list() does not recompile the pattern list on every call. This - may help if you are trying to optimize for speed, otherwise just use - the expect() method. This is called by expect(). If timeout==-1 then - the self.timeout value is used. If searchwindowsize==-1 then the - self.searchwindowsize value is used. """ - - return self.expect_loop(searcher_re(pattern_list), timeout, searchwindowsize) - - def expect_exact(self, pattern_list, timeout = -1, searchwindowsize = -1): - - """This is similar to expect(), but uses plain string matching instead - of compiled regular expressions in 'pattern_list'. The 'pattern_list' - may be a string; a list or other sequence of strings; or TIMEOUT and - EOF. - - This call might be faster than expect() for two reasons: string - searching is faster than RE matching and it is possible to limit the - search to just the end of the input buffer. - - This method is also useful when you don't want to have to worry about - escaping regular expression characters that you want to match.""" - - if not isinstance(pattern_list, list): - pattern_list = [pattern_list] - - for p in pattern_list: - if type(p) not in (str,) and p not in (TIMEOUT, EOF): - raise TypeError ('Argument must be one of StringTypes, EOF, TIMEOUT, or a list of those type. %s' % str(type(p))) - - return self.expect_loop(searcher_string(pattern_list), timeout, searchwindowsize) - - def expect_loop(self, searcher, timeout = -1, searchwindowsize = -1): - - """This is the common loop used inside expect. The 'searcher' should be - an instance of searcher_re or searcher_string, which describes how and what - to search for in the input. - - See expect() for other arguments, return value and exceptions. """ - - self.searcher = searcher - - if timeout == -1: - timeout = self.timeout - if timeout is not None: - end_time = time.time() + timeout - if searchwindowsize == -1: - searchwindowsize = self.searchwindowsize - - try: - incoming = self.buffer - freshlen = len(incoming) - while True: # Keep reading until exception or return. - index = searcher.search(incoming, freshlen, searchwindowsize) - if index >= 0: - self.buffer = incoming[searcher.end : ] - self.before = incoming[ : searcher.start] - self.after = incoming[searcher.start : searcher.end] - self.match = searcher.match - self.match_index = index - return self.match_index - # No match at this point - if timeout is not None and end_time < time.time(): - raise TIMEOUT ('Timeout exceeded in expect_any().') - # Still have time left, so read more data - c = self.read_nonblocking(self.maxread) - freshlen = len(c) - time.sleep (0.01) - incoming += c - except EOF as e: - self.buffer = '' - self.before = incoming - self.after = EOF - index = searcher.eof_index - if index >= 0: - self.match = EOF - self.match_index = index - return self.match_index - else: - self.match = None - self.match_index = None - raise EOF (str(e) + '\n' + str(self)) - except TIMEOUT as e: - self.buffer = incoming - self.before = incoming - self.after = TIMEOUT - index = searcher.timeout_index - if index >= 0: - self.match = TIMEOUT - self.match_index = index - return self.match_index - else: - self.match = None - self.match_index = None - raise TIMEOUT (str(e) + '\n' + str(self)) - except: - self.before = incoming - self.after = None - self.match = None - self.match_index = None - raise - - def getwinsize(self): - """This returns the terminal window size of the child tty. The return - value is a tuple of (rows, cols). """ - - return self.wtty.getwinsize() - - def setwinsize(self, r, c): - """Set the size of the child screen buffer. """ - - self.wtty.setwinsize(r, c) - - def interact(self): - """Makes the child console visible for interaction""" - - self.wtty.interact() - - def stop_interact(self): - """Hides the child console from the user.""" - - self.wtty.stop_interact() - -############################################################################## -# End of spawn_windows class -############################################################################## - -class Wtty: - - def __init__(self, timeout=30, codepage=None, echo=True): - self.__buffer = StringIO() - self.__bufferY = 0 - self.__currentReadCo = win32console.PyCOORDType(0, 0) - self.__consSize = [80, 16000] - self.__parentPid = 0 - self.__oproc = 0 - self.conpid = 0 - self.__otid = 0 - self.__switch = True - self.__childProcess = None - self.__conProcess = None - self.codepage = codepage - self.console = False - self.lastRead = 0 - self.lastReadData = "" - self.pid = None - self.processList = [] - self.__consout = None - # We need a timeout for connecting to the child process - self.timeout = timeout - self.totalRead = 0 - self.local_echo = echo - - def spawn(self, command, args=[], env=None): - """Spawns spawner.py with correct arguments.""" - - ts = time.time() - self.startChild(args, env) - - while True: - msg = win32gui.GetMessage(0, 0, 0) - childPid = msg[1][2] - # Sometimes win32gui.GetMessage returns a bogus PID, so keep calling it - # until we can successfully connect to the child or timeout is - # reached - if childPid: - try: - self.__childProcess = win32api.OpenProcess( - win32con.PROCESS_TERMINATE | win32con.PROCESS_QUERY_INFORMATION, False, childPid) - self.__conProcess = win32api.OpenProcess( - win32con.PROCESS_TERMINATE | win32con.PROCESS_QUERY_INFORMATION, False, self.conpid) - except pywintypes.error as e: - if time.time() > ts + self.timeout: - break - else: - self.pid = childPid - break - time.sleep(.05) - - if not self.__childProcess: - raise ExceptionPexpect ('The process ' + args[0] + ' could not be started.') - - - - winHandle = int(win32console.GetConsoleWindow()) - - self.__switch = True - - if winHandle != 0: - self.__parentPid = win32process.GetWindowThreadProcessId(winHandle)[1] - # Do we have a console attached? Do not rely on winHandle, because - # it will also be non-zero if we didn't have a console, and then - # spawned a child process! Using sys.stdout.isatty() seems safe - self.console = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() - # If the original process had a console, record a list of attached - # processes so we can check if we need to reattach/reallocate the - # console later - self.processList = win32console.GetConsoleProcessList() - else: - self.switchTo(False) - self.__switch = False - - def startChild(self, args, env): - si = win32process.GetStartupInfo() - si.dwFlags = win32process.STARTF_USESHOWWINDOW - si.wShowWindow = win32con.SW_HIDE - # Determine the directory of wexpect.py or, if we are running 'frozen' - # (eg. py2exe deployment), of the packed executable - dirname = os.path.dirname(sys.executable - if getattr(sys, 'frozen', False) else - os.path.abspath(__file__)) - if getattr(sys, 'frozen', False): - logdir = os.path.splitext(sys.executable)[0] - else: - logdir = dirname - logdir = os.path.basename(logdir) - spath = [dirname] - pyargs = ['-c'] - if getattr(sys, 'frozen', False): - # If we are running 'frozen', add library.zip and lib\library.zip - # to sys.path - # py2exe: Needs appropriate 'zipfile' option in setup script and - # 'bundle_files' 3 - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - if os.path.isdir(os.path.join(dirname, 'lib')): - dirname = os.path.join(dirname, 'lib') - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - pyargs.insert(0, '-S') # skip 'import site' - pid = win32process.GetCurrentProcessId() - tid = win32api.GetCurrentThreadId() - cp = self.codepage or windll.kernel32.GetACP() - # If we are running 'frozen', expect python.exe in the same directory - # as the packed executable. - # py2exe: The python executable can be included via setup script by - # adding it to 'data_files' - commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') - if getattr(sys, 'frozen', False) else - os.path.join(os.path.dirname(sys.executable), 'python.exe'), - ' '.join(pyargs), - "import sys; sys.path = %r + sys.path;" - "args = %r; import wexpect;" - "wexpect.ConsoleReader(wexpect.join_args(args), %i, %i, cp=%i, logdir=%r)" % (spath, args, pid, tid, cp, logdir)) - - - self.__oproc, _, self.conpid, self.__otid = win32process.CreateProcess(None, commandLine, None, None, False, - win32process.CREATE_NEW_CONSOLE, env, None, si) - - - def switchTo(self, attatched=True): - """Releases from the current console and attatches - to the childs.""" - - if not self.__switch: - return - - try: - # No 'attached' check is needed, FreeConsole() can be called multiple times. - win32console.FreeConsole() - # This is the workaround for #14. The #14 will still occure if the child process - # finishes between this `isalive()` check and `AttachConsole(self.conpid)`. (However the - # risk is low.) - if not self.isalive(console=True): - # When child has finished... - raise EOF('End Of File (EOF) in switchTo().') - - win32console.AttachConsole(self.conpid) - self.__consin = win32console.GetStdHandle(win32console.STD_INPUT_HANDLE) - self.__consout = self.getConsoleOut() - - except pywintypes.error as e: - # pywintypes.error: (5, 'AttachConsole', 'Access is denied.') - # When child has finished... - logging.info(e) - # In case of any error: We "switch back" (attach) our original console, then raise the - # error. - self.switchBack() - raise EOF('End Of File (EOF) in switchTo().') - except: - # In case of any error: We "switch back" (attach) our original console, then raise the - # error. - self.switchBack() - raise - - - def switchBack(self): - """Releases from the current console and attaches - to the parents.""" - - if not self.__switch: - return - - if self.console: - # If we originally had a console, re-attach it (or allocate a new one) - # If we didn't have a console to begin with, there's no need to - # re-attach/allocate - win32console.FreeConsole() - if len(self.processList) > 1: - # Our original console is still present, re-attach - win32console.AttachConsole(self.__parentPid) - else: - # Our original console has been free'd, allocate a new one - win32console.AllocConsole() - - self.__consin = None - self.__consout = None - - def getConsoleOut(self): - consout = win32file.CreateFile('CONOUT$', - win32con.GENERIC_READ | win32con.GENERIC_WRITE, - win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE, - None, - win32con.OPEN_EXISTING, - 0, - 0) - - return win32console.PyConsoleScreenBufferType(consout) - - def getchild(self): - """Returns a handle to the child process.""" - - return self.__childProcess - - def terminate_child(self): - """Terminate the child process.""" - win32api.TerminateProcess(self.__childProcess, 1) - # win32api.win32process.TerminateProcess(self.__childProcess, 1) - - def createKeyEvent(self, char): - """Creates a single key record corrosponding to - the ascii character char.""" - - evt = win32console.PyINPUT_RECORDType(win32console.KEY_EVENT) - evt.KeyDown = True - evt.Char = char - evt.RepeatCount = 1 - return evt - - def write(self, s): - """Writes input into the child consoles input buffer.""" - - if len(s) == 0: - return 0 - self.switchTo() - try: - if s[-1] == '\n': - s = s[:-1] - records = [self.createKeyEvent(c) for c in str(s)] - if not self.__consout: - return "" - - # Store the current cursor position to hide characters in local echo disabled mode (workaround). - consinfo = self.__consout.GetConsoleScreenBufferInfo() - startCo = consinfo['CursorPosition'] - - # Send the string to console input - wrote = self.__consin.WriteConsoleInput(records) - - # Wait until all input has been recorded by the console. - ts = time.time() - while self.__consin.PeekConsoleInput(8) != (): - if time.time() > ts + len(s) * .1 + .5: - break - time.sleep(.05) - - # Hide characters in local echo disabled mode (workaround). - if not self.local_echo: - self.__consout.FillConsoleOutputCharacter(screenbufferfillchar, len(s), startCo) - - return wrote - finally: - self.switchBack() - - def getCoord(self, offset): - """Converts an offset to a point represented as a tuple.""" - - x = offset % self.__consSize[0] - y = offset // self.__consSize[0] - return win32console.PyCOORDType(x, y) - - def getOffset(self, coord): - """Converts a tuple-point to an offset.""" - - return coord.X + coord.Y * self.__consSize[0] - - def readConsole(self, startCo, endCo): - """Reads the console area from startCo to endCo and returns it - as a string.""" - - buff = [] - self.lastRead = 0 - - while True: - startOff = self.getOffset(startCo) - endOff = self.getOffset(endCo) - readlen = endOff - startOff - - if readlen <= 0: - break - - if readlen > 4000: - readlen = 4000 - endPoint = self.getCoord(startOff + readlen) - - s = self.__consout.ReadConsoleOutputCharacter(readlen, startCo) - self.lastRead += len(s) - self.totalRead += len(s) - buff.append(s) - - startCo = endPoint - - return ''.join(buff) - - def parseData(self, s): - """Ensures that special characters are interpretted as - newlines or blanks, depending on if there written over - characters or screen-buffer-fill characters.""" - - strlist = [] - for i, c in enumerate(s): - if c == screenbufferfillchar: - if (self.totalRead - self.lastRead + i + 1) % self.__consSize[0] == 0: - strlist.append('\r\n') - else: - strlist.append(c) - - s = ''.join(strlist) - return s - - - def readConsoleToCursor(self): - """Reads from the current read position to the current cursor - position and inserts the string into self.__buffer.""" - - if not self.__consout: - return "" - - consinfo = self.__consout.GetConsoleScreenBufferInfo() - cursorPos = consinfo['CursorPosition'] - - logger.debug('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) - - isSameX = cursorPos.X == self.__currentReadCo.X - isSameY = cursorPos.Y == self.__currentReadCo.Y - isSamePos = isSameX and isSameY - - logger.debug('isSameY: %r' % isSameY) - logger.debug('isSamePos: %r' % isSamePos) - - if isSameY or not self.lastReadData.endswith('\r\n'): - # Read the current slice again - self.totalRead -= self.lastRead - self.__currentReadCo.X = 0 - self.__currentReadCo.Y = self.__bufferY - - logger.debug('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) - - raw = self.readConsole(self.__currentReadCo, cursorPos) - rawlist = [] - while raw: - rawlist.append(raw[:self.__consSize[0]]) - raw = raw[self.__consSize[0]:] - raw = ''.join(rawlist) - s = self.parseData(raw) - logger.debug(s) - for i, line in enumerate(reversed(rawlist)): - if line.endswith(screenbufferfillchar): - # Record the Y offset where the most recent line break was detected - self.__bufferY += len(rawlist) - i - break - - logger.debug('lastReadData: %r' % self.lastReadData) - logger.debug('s: %r' % s) - - if isSamePos and self.lastReadData == s: - logger.debug('isSamePos and self.lastReadData == s') - s = '' - - logger.debug('s: %r' % s) - - if s: - lastReadData = self.lastReadData - pos = self.getOffset(self.__currentReadCo) - self.lastReadData = s - if isSameY or not lastReadData.endswith('\r\n'): - # Detect changed lines - self.__buffer.seek(pos) - buf = self.__buffer.read() - logger.debug('buf: %r' % buf) - logger.debug('raw: %r' % raw) - if raw.startswith(buf): - # Line has grown - rawslice = raw[len(buf):] - # Update last read bytes so line breaks can be detected in parseData - lastRead = self.lastRead - self.lastRead = len(rawslice) - s = self.parseData(rawslice) - self.lastRead = lastRead - else: - # Cursor has been repositioned - s = '\r' + s - logger.debug('s: %r' % s) - self.__buffer.seek(pos) - self.__buffer.truncate() - self.__buffer.write(raw) - - self.__currentReadCo.X = cursorPos.X - self.__currentReadCo.Y = cursorPos.Y - - return s - - - def read_nonblocking(self, size): - """Reads data from the console if available, otherwise - returns empty string""" - - try: - self.switchTo() - time.sleep(.01) - - if self.__currentReadCo.Y > maxconsoleY: - time.sleep(.2) - - s = self.readConsoleToCursor() - - if self.__currentReadCo.Y > maxconsoleY: - self.refreshConsole() - - return s - - finally: - self.switchBack() - - raise Exception('Unreachable code...') # pragma: no cover - - - def refreshConsole(self): - """Clears the console after pausing the child and - reading all the data currently on the console.""" - - orig = win32console.PyCOORDType(0, 0) - self.__consout.SetConsoleCursorPosition(orig) - self.__currentReadCo.X = 0 - self.__currentReadCo.Y = 0 - writelen = self.__consSize[0] * self.__consSize[1] - # Use NUL as fill char because it displays as whitespace - # (if we interact() with the child) - self.__consout.FillConsoleOutputCharacter(screenbufferfillchar, writelen, orig) - - self.__bufferY = 0 - self.__buffer.truncate(0) - - consinfo = self.__consout.GetConsoleScreenBufferInfo() - cursorPos = consinfo['CursorPosition'] - logger.debug('refreshConsole: cursorPos %s' % cursorPos) - - - def setecho(self, state): - """Sets the echo mode of the child console. - This is a workaround of the setecho. The original GetConsoleMode() / SetConsoleMode() - methods didn't work. See git history for the concrete implementation. - 2020.01.09 raczben - """ - - self.local_echo = state - - def getecho(self): - """Returns the echo mode of the child console. - This is a workaround of the getecho. The original GetConsoleMode() / SetConsoleMode() - methods didn't work. See git history for the concrete implementation. - 2020.01.09 raczben - """ - - return self.local_echo - - def getwinsize(self): - """Returns the size of the child console as a tuple of - (rows, columns).""" - - self.switchTo() - try: - size = self.__consout.GetConsoleScreenBufferInfo()['Size'] - finally: - self.switchBack() - return (size.Y, size.X) - - def setwinsize(self, r, c): - """Sets the child console screen buffer size to (r, c).""" - - self.switchTo() - try: - self.__consout.SetConsoleScreenBufferSize(win32console.PyCOORDType(c, r)) - finally: - self.switchBack() - - def interact(self): - """Displays the child console for interaction.""" - - self.switchTo() - try: - win32gui.ShowWindow(win32console.GetConsoleWindow(), win32con.SW_SHOW) - finally: - self.switchBack() - - def stop_interact(self): - """Hides the child console.""" - - self.switchTo() - try: - win32gui.ShowWindow(win32console.GetConsoleWindow(), win32con.SW_HIDE) - finally: - self.switchBack() - - def isalive(self, console=False): - """True if the child is still alive, false otherwise""" - - if console: - return win32process.GetExitCodeProcess(self.__conProcess) == win32con.STILL_ACTIVE - else: - return win32process.GetExitCodeProcess(self.__childProcess) == win32con.STILL_ACTIVE - -class ConsoleReader: # pragma: no cover - - def __init__(self, path, pid, tid, env = None, cp=None, logdir=None): - self.logdir = logdir - logger.debug('=' * 80, 'consolereader', logdir) - logger.debug("OEM code page: %s" % windll.kernel32.GetOEMCP(), 'consolereader', logdir) - logger.debug("ANSI code page: %s" % windll.kernel32.GetACP(), 'consolereader', logdir) - logger.debug("Console output code page: %s" % windll.kernel32.GetConsoleOutputCP(), 'consolereader', logdir) - if cp: - logger.debug("Setting console output code page to %s" % cp, 'consolereader', logdir) - try: - win32console.SetConsoleOutputCP(cp) - except Exception as e: - logger.debug(e, 'consolereader', logdir) - else: - logger.debug("Console output code page: %s" % windll.kernel32.GetConsoleOutputCP(), 'consolereader', logdir) - logger.debug('Spawning %s' % path, 'consolereader', logdir) - try: - try: - consout = self.getConsoleOut() - self.initConsole(consout) - - si = win32process.GetStartupInfo() - self.__childProcess, _, childPid, self.__tid = win32process.CreateProcess(None, path, None, None, False, - 0, None, None, si) - except Exception as e: - logger.debug(e, 'consolereader', logdir) - time.sleep(.1) - win32api.PostThreadMessage(int(tid), win32con.WM_USER, 0, 0) - sys.exit() - - time.sleep(.1) - - win32api.PostThreadMessage(int(tid), win32con.WM_USER, childPid, 0) - - parent = win32api.OpenProcess(win32con.PROCESS_TERMINATE | win32con.PROCESS_QUERY_INFORMATION , 0, int(pid)) - paused = False - - while True: - consinfo = consout.GetConsoleScreenBufferInfo() - cursorPos = consinfo['CursorPosition'] - - if win32process.GetExitCodeProcess(parent) != win32con.STILL_ACTIVE or win32process.GetExitCodeProcess(self.__childProcess) != win32con.STILL_ACTIVE: - time.sleep(.1) - try: - win32process.TerminateProcess(self.__childProcess, 0) - except pywintypes.error as e: - # 'Access denied' happens always? Perhaps if not - # running as admin (or UAC enabled under Vista/7). - # Don't log. Child process will exit regardless when - # calling sys.exit - if e.args[0] != winerror.ERROR_ACCESS_DENIED: - logger.debug(e, 'consolereader', logdir) - sys.exit() - - if cursorPos.Y > maxconsoleY and not paused: - self.suspendThread() - paused = True - - if cursorPos.Y <= maxconsoleY and paused: - self.resumeThread() - paused = False - - time.sleep(.1) - except Exception as e: - logger.debug(e, 'consolereader', logdir) - time.sleep(.1) - - - def handler(self, sig): - logger.debug(sig, 'consolereader', logdir) - return False - - def getConsoleOut(self): - consout = win32file.CreateFile('CONOUT$', - win32con.GENERIC_READ | win32con.GENERIC_WRITE, - win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE, - None, - win32con.OPEN_EXISTING, - 0, - 0) - - return win32console.PyConsoleScreenBufferType(consout) - - def initConsole(self, consout): - rect = win32console.PySMALL_RECTType(0, 0, 79, 24) - consout.SetConsoleWindowInfo(True, rect) - size = win32console.PyCOORDType(80, 16000) - consout.SetConsoleScreenBufferSize(size) - pos = win32console.PyCOORDType(0, 0) - # Use NUL as fill char because it displays as whitespace - # (if we interact() with the child) - consout.FillConsoleOutputCharacter(screenbufferfillchar, size.X * size.Y, pos) - - def suspendThread(self): - """Pauses the main thread of the child process.""" - - handle = windll.kernel32.OpenThread(win32con.THREAD_SUSPEND_RESUME, 0, self.__tid) - win32process.SuspendThread(handle) - - def resumeThread(self): - """Un-pauses the main thread of the child process.""" - - handle = windll.kernel32.OpenThread(win32con.THREAD_SUSPEND_RESUME, 0, self.__tid) - win32process.ResumeThread(handle) - -class searcher_string (object): - - """This is a plain string search helper for the spawn.expect_any() method. - - Attributes: - - eof_index - index of EOF, or -1 - timeout_index - index of TIMEOUT, or -1 - - After a successful match by the search() method the following attributes - are available: - - start - index into the buffer, first byte of match - end - index into the buffer, first byte after match - match - the matching string itself - """ - - def __init__(self, strings): - - """This creates an instance of searcher_string. This argument 'strings' - may be a list; a sequence of strings; or the EOF or TIMEOUT types. """ - - self.eof_index = -1 - self.timeout_index = -1 - self._strings = [] - for n, s in zip(list(range(len(strings))), strings): - if s is EOF: - self.eof_index = n - continue - if s is TIMEOUT: - self.timeout_index = n - continue - self._strings.append((n, s)) - - def __str__(self): - - """This returns a human-readable string that represents the state of - the object.""" - - ss = [ (ns[0],' %d: "%s"' % ns) for ns in self._strings ] - ss.append((-1,'searcher_string:')) - if self.eof_index >= 0: - ss.append ((self.eof_index,' %d: EOF' % self.eof_index)) - if self.timeout_index >= 0: - ss.append ((self.timeout_index,' %d: TIMEOUT' % self.timeout_index)) - ss.sort() - ss = list(zip(*ss))[1] - return '\n'.join(ss) - - def search(self, buffer, freshlen, searchwindowsize=None): - - """This searches 'buffer' for the first occurence of one of the search - strings. 'freshlen' must indicate the number of bytes at the end of - 'buffer' which have not been searched before. It helps to avoid - searching the same, possibly big, buffer over and over again. - - See class spawn for the 'searchwindowsize' argument. - - If there is a match this returns the index of that string, and sets - 'start', 'end' and 'match'. Otherwise, this returns -1. """ - - absurd_match = len(buffer) - first_match = absurd_match - - # 'freshlen' helps a lot here. Further optimizations could - # possibly include: - # - # using something like the Boyer-Moore Fast String Searching - # Algorithm; pre-compiling the search through a list of - # strings into something that can scan the input once to - # search for all N strings; realize that if we search for - # ['bar', 'baz'] and the input is '...foo' we need not bother - # rescanning until we've read three more bytes. - # - # Sadly, I don't know enough about this interesting topic. /grahn - - for index, s in self._strings: - if searchwindowsize is None: - # the match, if any, can only be in the fresh data, - # or at the very end of the old data - offset = -(freshlen+len(s)) - else: - # better obey searchwindowsize - offset = -searchwindowsize - n = buffer.find(s, offset) - if n >= 0 and n < first_match: - first_match = n - best_index, best_match = index, s - if first_match == absurd_match: - return -1 - self.match = best_match - self.start = first_match - self.end = self.start + len(self.match) - return best_index - -class searcher_re (object): - - """This is regular expression string search helper for the - spawn.expect_any() method. - - Attributes: - - eof_index - index of EOF, or -1 - timeout_index - index of TIMEOUT, or -1 - - After a successful match by the search() method the following attributes - are available: - - start - index into the buffer, first byte of match - end - index into the buffer, first byte after match - match - the re.match object returned by a succesful re.search - - """ - - def __init__(self, patterns): - - """This creates an instance that searches for 'patterns' Where - 'patterns' may be a list or other sequence of compiled regular - expressions, or the EOF or TIMEOUT types.""" - - self.eof_index = -1 - self.timeout_index = -1 - self._searches = [] - for n, s in zip(list(range(len(patterns))), patterns): - if s is EOF: - self.eof_index = n - continue - if s is TIMEOUT: - self.timeout_index = n - continue - self._searches.append((n, s)) - - def __str__(self): - - """This returns a human-readable string that represents the state of - the object.""" - - ss = [ (n,' %d: re.compile("%s")' % (n,str(s.pattern))) for n,s in self._searches] - ss.append((-1,'searcher_re:')) - if self.eof_index >= 0: - ss.append ((self.eof_index,' %d: EOF' % self.eof_index)) - if self.timeout_index >= 0: - ss.append ((self.timeout_index,' %d: TIMEOUT' % self.timeout_index)) - ss.sort() - ss = list(zip(*ss))[1] - return '\n'.join(ss) - - def search(self, buffer, freshlen, searchwindowsize=None): - - """This searches 'buffer' for the first occurence of one of the regular - expressions. 'freshlen' must indicate the number of bytes at the end of - 'buffer' which have not been searched before. - - See class spawn for the 'searchwindowsize' argument. - - If there is a match this returns the index of that string, and sets - 'start', 'end' and 'match'. Otherwise, returns -1.""" - - absurd_match = len(buffer) - first_match = absurd_match - # 'freshlen' doesn't help here -- we cannot predict the - # length of a match, and the re module provides no help. - if searchwindowsize is None: - searchstart = 0 - else: - searchstart = max(0, len(buffer)-searchwindowsize) - for index, s in self._searches: - match = s.search(buffer, searchstart) - if match is None: - continue - n = match.start() - if n < first_match: - first_match = n - the_match = match - best_index = index - if first_match == absurd_match: - return -1 - self.start = first_match - self.match = the_match - self.end = self.match.end() - return best_index - - -def join_args(args): - """Joins arguments into a command line. It quotes all arguments that contain - spaces or any of the characters ^!$%&()[]{}=;'+,`~""" - commandline = [] - for arg in args: - if re.search('[\^!$%&()[\]{}=;\'+,`~\s]', arg): - arg = '"%s"' % arg - commandline.append(arg) - return ' '.join(commandline) - -def split_command_line(command_line, escape_char = '^'): - """This splits a command line into a list of arguments. It splits arguments - on spaces, but handles embedded quotes, doublequotes, and escaped - characters. It's impossible to do this with a regular expression, so I - wrote a little state machine to parse the command line. """ - - arg_list = [] - arg = '' - - # Constants to name the states we can be in. - state_basic = 0 - state_esc = 1 - state_singlequote = 2 - state_doublequote = 3 - state_whitespace = 4 # The state of consuming whitespace between commands. - state = state_basic - - for c in command_line: - if state == state_basic or state == state_whitespace: - if c == escape_char: # Escape the next character - state = state_esc - elif c == r"'": # Handle single quote - state = state_singlequote - elif c == r'"': # Handle double quote - state = state_doublequote - elif c.isspace(): - # Add arg to arg_list if we aren't in the middle of whitespace. - if state == state_whitespace: - None # Do nothing. - else: - arg_list.append(arg) - arg = '' - state = state_whitespace - else: - arg = arg + c - state = state_basic - elif state == state_esc: - arg = arg + c - state = state_basic - elif state == state_singlequote: - if c == r"'": - state = state_basic - else: - arg = arg + c - elif state == state_doublequote: - if c == r'"': - state = state_basic - else: - arg = arg + c - - if arg != '': - arg_list.append(arg) - return arg_list diff --git a/wexpect/__init__.py b/wexpect/__init__.py new file mode 100644 index 0000000..02d6f64 --- /dev/null +++ b/wexpect/__init__.py @@ -0,0 +1,16 @@ +# __init__.py + +from .wexpect_util import split_command_line +from .wexpect_util import join_args +from .wexpect_util import ExceptionPexpect +from .wexpect_util import EOF +from .wexpect_util import TIMEOUT + +from .console_reader import ConsoleReaderSocket +from .console_reader import ConsoleReaderPipe + +from .spawn import Spawn +from .spawn import Spawn as spawn + +__all__ = ['split_command_line', 'join_args', 'ExceptionPexpect', 'EOF', 'TIMEOUT', + 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'Spawn'] diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 4d270a4..cf9cb39 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -1,34 +1,65 @@ +"""Wexpect is a Windows variant of pexpect https://pexpect.readthedocs.io. + +Wexpect is a Python module for spawning child applications and controlling +them automatically. + +console_reader Implements a virtual terminal, and starts the child program. +The main wexpect.Spawn class connect to this class to reach the child's terminal. + +Credits: Noah Spurrier, Richard Holden, Marco Molteni, Kimberley Burchett, +Robert Stone, Hartmut Goebel, Chad Schroeder, Erick Tryzelaar, Dave Kirby, Ids +vander Molen, George Todd, Noel Taylor, Nicolas D. Cesar, Alexander Gattin, +Geoffrey Marshall, Francisco Lourenco, Glen Mabey, Karthik Gurusamy, Fernando +Perez, Corey Minyard, Jon Cohen, Guillaume Chazarain, Andrew Ryan, Nick +Craig-Wood, Andrew Stone, Jorgen Grahn, Benedek Racz + +Free, open source, and all that good stuff. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Wexpect Copyright (c) 2019 Benedek Racz + +""" + import time -import sys import logging import os -import re -import shutil -import types import traceback -import signal import pkg_resources from io import StringIO -from ctypes import windll +import ctypes import pywintypes -from win32com.shell.shell import SHGetSpecialFolderPath import win32console import win32process import win32con -import win32gui -import win32api import win32file import winerror import win32pipe - -__version__ = 'test' +import socket # # System-wide constants # screenbufferfillchar = '\4' maxconsoleY = 8000 +default_port = 4321 # The version is handled by the package: pbr, which derives the version from the git tags. try: @@ -52,25 +83,19 @@ try: except KeyError: logger.setLevel(logging.ERROR) -# Test the logger -#logger.info('wexpect imported; logger working') - - -class ConsoleReader: +class ConsoleReaderBase: """Consol class (aka. client-side python class) for the child. This class initialize the console starts the child in it and reads the console periodically. """ - def __init__(self, path, parent_pid, parent_tid, cp=None): - """Initialize the console starts the child in it and reads the console periodically. - - + def __init__(self, path, parent_pid, cp=None, window_size_x=80, window_size_y=25, + buffer_size_x=80, buffer_size_y=16000, **kwargs): + """Initialize the console starts the child in it and reads the console periodically. Args: path (str): Child's executable with arguments. parent_pid (int): Parent (aka. host) process process-ID - parent_tid (int): Parent (aka. host) process thread-ID cp (:obj:, optional): Output console code page. """ self.lastRead = 0 @@ -79,25 +104,27 @@ class ConsoleReader: self.totalRead = 0 self.__buffer = StringIO() self.__currentReadCo = win32console.PyCOORDType(0, 0) - + self.pipe = None + self.connection = None + self.consin = None + self.consout = None + self.local_echo = True logger.info("ConsoleReader started") - logger.info("parent_tid %s" % parent_tid) - self.create_pipe() + if cp: try: logger.info("Setting console output code page to %s" % cp) win32console.SetConsoleOutputCP(cp) - logger.info("Console output code page: %s" % windll.kernel32.GetConsoleOutputCP()) + logger.info("Console output code page: %s" % ctypes.windll.kernel32.GetConsoleOutputCP()) except Exception as e: logger.info(e) try: + self.create_connection(**kwargs) logger.info('Spawning %s' % path) try: self.initConsole() - - time.sleep(1) si = win32process.GetStartupInfo() self.__childProcess, _, childPid, self.__tid = win32process.CreateProcess(None, path, None, None, False, 0, None, None, si) @@ -111,14 +138,18 @@ class ConsoleReader: time.sleep(.1) return - time.sleep(.1) + time.sleep(.2) + self.write('ls') + self.write(os.linesep) paused = False - while True: consinfo = self.consout.GetConsoleScreenBufferInfo() cursorPos = consinfo['CursorPosition'] + self.send_to_host(self.readConsoleToCursor()) + s = self.get_from_host() + self.write(s) if win32process.GetExitCodeProcess(self.__childProcess) != win32con.STILL_ACTIVE: time.sleep(.1) @@ -131,6 +162,10 @@ class ConsoleReader: """ if e.args[0] != winerror.ERROR_ACCESS_DENIED: logger.info(e) + + time.sleep(.1) + self.send_to_host(self.readConsoleToCursor()) + time.sleep(.1) return if cursorPos.Y > maxconsoleY and not paused: @@ -144,36 +179,63 @@ class ConsoleReader: paused = False time.sleep(.1) - except Exception as e: - logger.error(e) + except: + logger.error(traceback.format_exc()) time.sleep(.1) + finally: + self.close_connection() - def create_pipe(self): - pid = win32process.GetCurrentProcessId() - pipe_name = 'wexpect_pipe_c2s_{}'.format(pid) - pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) - logger.info('Start pipe server: %s', pipe_full_path) - self.pipe = win32pipe.CreateNamedPipe( - pipe_full_path, - win32pipe.PIPE_ACCESS_DUPLEX, - win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT, - 1, 65536, 65536, 0, None) - logger.info("waiting for client") - win32pipe.ConnectNamedPipe(self.pipe, None) - logger.info('got client') + def write(self, s): + """Writes input into the child consoles input buffer.""" + + if len(s) == 0: + return 0 + if s[-1] == '\n': + s = s[:-1] + records = [self.createKeyEvent(c) for c in str(s)] + if not self.consout: + return "" - def write_pipe(self, msg): - # convert to bytes - msg_bytes = str.encode(msg) - win32file.WriteFile(self.pipe, msg_bytes) + # Store the current cursor position to hide characters in local echo disabled mode (workaround). + consinfo = self.consout.GetConsoleScreenBufferInfo() + startCo = consinfo['CursorPosition'] + + # Send the string to console input + wrote = self.consin.WriteConsoleInput(records) + + # Wait until all input has been recorded by the console. + ts = time.time() + while self.consin.PeekConsoleInput(8) != (): + if time.time() > ts + len(s) * .1 + .5: + break + time.sleep(.05) - def initConsole(self, consout=None): + # Hide characters in local echo disabled mode (workaround). + if not self.local_echo: + self.consout.FillConsoleOutputCharacter(screenbufferfillchar, len(s), startCo) + + return wrote + + def createKeyEvent(self, char): + """Creates a single key record corrosponding to + the ascii character char.""" + + evt = win32console.PyINPUT_RECORDType(win32console.KEY_EVENT) + evt.KeyDown = True + evt.Char = char + evt.RepeatCount = 1 + return evt + + def initConsole(self, consout=None, window_size_x=80, window_size_y=25, buffer_size_x=80, + buffer_size_y=16000): if not consout: consout=self.getConsoleOut() - rect = win32console.PySMALL_RECTType(0, 0, 79, 24) + self.consin = win32console.GetStdHandle(win32console.STD_INPUT_HANDLE) + + rect = win32console.PySMALL_RECTType(0, 0, window_size_x-1, window_size_y-1) consout.SetConsoleWindowInfo(True, rect) - size = win32console.PyCOORDType(80, 16000) + size = win32console.PyCOORDType(buffer_size_x, buffer_size_y) consout.SetConsoleScreenBufferSize(size) pos = win32console.PyCOORDType(0, 0) # Use NUL as fill char because it displays as whitespace @@ -238,11 +300,6 @@ class ConsoleReader: consinfo = self.consout.GetConsoleScreenBufferInfo() endCo = consinfo['CursorPosition'] endCo= self.getCoord(0 + self.getOffset(endCo)) - # endCo.Y = endCo.Y+1 - # logger.info(endCo.Y+1) - - logger.info(startCo) - logger.info(endCo) buff = [] self.lastRead = 0 @@ -266,7 +323,6 @@ class ConsoleReader: startCo = endPoint - logger.info(repr(s)) return ''.join(buff) @@ -304,7 +360,6 @@ class ConsoleReader: raw = raw[self.__consSize.X:] raw = ''.join(rawlist) s = self.parseData(raw) - logger.debug(s) for i, line in enumerate(reversed(rawlist)): if line.endswith(screenbufferfillchar): # Record the Y offset where the most recent line break was detected @@ -318,8 +373,6 @@ class ConsoleReader: logger.debug('isSamePos and self.lastReadData == s') s = '' - logger.debug('s: %r' % s) - if s: lastReadData = self.lastReadData pos = self.getOffset(self.__currentReadCo) @@ -328,8 +381,6 @@ class ConsoleReader: # Detect changed lines self.__buffer.seek(pos) buf = self.__buffer.read() - logger.debug('buf: %r' % buf) - logger.debug('raw: %r' % raw) if raw.startswith(buf): # Line has grown rawslice = raw[len(buf):] @@ -340,8 +391,7 @@ class ConsoleReader: self.lastRead = lastRead else: # Cursor has been repositioned - s = '\r' + s - logger.debug('s: %r' % s) + s = '\r' + s self.__buffer.seek(pos) self.__buffer.truncate() self.__buffer.write(raw) @@ -351,88 +401,80 @@ class ConsoleReader: return s +class ConsoleReaderSocket(ConsoleReaderBase): -def client(path, pid, tid): - try: - w = ConsoleReader(path, pid, tid) - time.sleep(1) - w.write_pipe(w.readConsoleToCursor()) - except Exception: - tb = traceback.format_exc() - logger.error(tb) - - -def pipe_client(conpid): - pipe_name = 'wexpect_pipe_c2s_{}'.format(conpid) - pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) - print('Trying to connect to pipe: {}'.format(pipe_full_path)) - quit = False - - while not quit: + + def create_connection(self, **kwargs): + + self.port = kwargs['port'] + # Create a TCP/IP socket + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_address = ('localhost', self.port) + self.sock.bind(server_address) + logger.info(f'Socket started at port: {self.port}') + + # Listen for incoming connections + self.sock.listen(1) + self.connection, client_address = self.sock.accept() + self.connection.settimeout(.2) + logger.info(f'Client connected: {client_address}') + + def close_connection(self): + if self.connection: + self.connection.close() + + def send_to_host(self, msg): + # convert to bytes + msg_bytes = str.encode(msg) + self.connection.sendall(msg_bytes) + + def get_from_host(self): try: - handle = win32file.CreateFile( - pipe_full_path, - win32file.GENERIC_READ | win32file.GENERIC_WRITE, - 0, - None, - win32file.OPEN_EXISTING, - 0, - None - ) - res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) - if res == 0: - print(f"SetNamedPipeHandleState return code: {res}") - while True: - resp = win32file.ReadFile(handle, 64*1024) - print(f"message: {resp}") - except pywintypes.error as e: - if e.args[0] == 2: - print("no pipe, trying again in a bit") - time.sleep(0.2) - elif e.args[0] == 109: - print("broken pipe, bye bye") - quit = True - - -def main(): - pass + msg = self.connection.recv(4096) + except socket.timeout as e: + err = e.args[0] + # this next if/else is a bit redundant, but illustrates how the + # timeout exception is setup + if err == 'timed out': + logger.debug('recv timed out, retry later') + return '' + else: + raise + else: + if len(msg) == 0: + raise Exception('orderly shutdown on server end') + else: + # got a message do something :) + return msg.decode() + + +class ConsoleReaderPipe(ConsoleReaderBase): + def create_connection(self): + pid = win32process.GetCurrentProcessId() + pipe_name = 'wexpect_{}'.format(pid) + pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) + logger.info('Start pipe server: %s', pipe_full_path) + self.pipe = win32pipe.CreateNamedPipe( + pipe_full_path, + win32pipe.PIPE_ACCESS_DUPLEX, + win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT, + 1, 65536, 65536, 0, None) + logger.info("waiting for client") + win32pipe.ConnectNamedPipe(self.pipe, None) + logger.info('got client') - -if __name__ == '__main__': - - si = win32process.GetStartupInfo() - si.dwFlags = win32process.STARTF_USESHOWWINDOW - si.wShowWindow = win32con.SW_HIDE - pyargs = ['-c'] - - dirname = os.path.dirname(sys.executable - if getattr(sys, 'frozen', False) else - os.path.abspath(__file__)) - -# client('uname') - - pid = win32process.GetCurrentProcessId() - tid = win32api.GetCurrentThreadId() - - commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') - if getattr(sys, 'frozen', False) else - os.path.join(os.path.dirname(sys.executable), 'python.exe'), - ' '.join(pyargs), - "import sys;" - "sys.path.append('D:\\\\bt\\\\wexpect');" - "import console_reader;" - "import time;" - "console_reader.client('uname', {tid}, {pid});".format(pid=pid, tid=tid) - ) - + def close_connection(self): + if self.pipe: + raise Exception(f'Unimplemented close') - print(commandLine) + def send_to_host(self, msg): + # convert to bytes + msg_bytes = str.encode(msg) + win32file.WriteFile(self.pipe, msg_bytes) - __oproc, _, conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, - win32process.CREATE_NEW_CONSOLE, None, None, si) -# time.sleep(3) - pipe_client(conpid) - - - time.sleep(5) - \ No newline at end of file + def get_from_host(self): + resp = win32file.ReadFile(self.pipe, 64*1024) + ret = resp[1] + return ret + + \ No newline at end of file diff --git a/wexpect/spawn.py b/wexpect/spawn.py new file mode 100644 index 0000000..6e96e34 --- /dev/null +++ b/wexpect/spawn.py @@ -0,0 +1,849 @@ +"""Wexpect is a Windows variant of pexpect https://pexpect.readthedocs.io. + +Wexpect is a Python module for spawning child applications and controlling +them automatically. Wexpect can be used for automating interactive applications +such as ssh, ftp, passwd, telnet, etc. It can be used to a automate setup +scripts for duplicating software package installations on different servers. It +can be used for automated software testing. Wexpect is in the spirit of Don +Libes' Expect, but Wexpect is pure Python. Other Expect-like modules for Python +require TCL and Expect or require C extensions to be compiled. Wexpect does not +use C, Expect, or TCL extensions. + +There are two main interfaces to Wexpect -- the function, run() and the class, +spawn. You can call the run() function to execute a command and return the +output. This is a handy replacement for os.system(). + +For example:: + + wexpect.run('ls -la') + +The more powerful interface is the spawn class. You can use this to spawn an +external child command and then interact with the child by sending lines and +expecting responses. + +For example:: + + child = wexpect.spawn('scp foo myname@host.example.com:.') + child.expect('Password:') + child.sendline(mypassword) + +This works even for commands that ask for passwords or other input outside of +the normal stdio streams. + +Spawn file is the main (aka. host) class of the wexpect. The user call Spawn, which +start the console_reader as a subprocess, which starts the read child. + +Credits: Noah Spurrier, Richard Holden, Marco Molteni, Kimberley Burchett, +Robert Stone, Hartmut Goebel, Chad Schroeder, Erick Tryzelaar, Dave Kirby, Ids +vander Molen, George Todd, Noel Taylor, Nicolas D. Cesar, Alexander Gattin, +Geoffrey Marshall, Francisco Lourenco, Glen Mabey, Karthik Gurusamy, Fernando +Perez, Corey Minyard, Jon Cohen, Guillaume Chazarain, Andrew Ryan, Nick +Craig-Wood, Andrew Stone, Jorgen Grahn, Benedek Racz + +Free, open source, and all that good stuff. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Wexpect Copyright (c) 2019 Benedek Racz + +""" + +import time +import sys +import os +import shutil +import re +import traceback + +import pywintypes +import win32process +import win32con +import win32api +import win32file +import winerror +import win32pipe +import socket + +from wexpect_util import ExceptionPexpect +from wexpect_util import EOF +from wexpect_util import TIMEOUT +from wexpect_util import split_command_line + + +class Spawn: + def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, + logfile=None, cwd=None, env=None, codepage=None, echo=True): + """This starts the given command in a child process. This does all the + fork/exec type of stuff for a pty. This is called by __init__. If args + is empty then command will be parsed (split on spaces) and args will be + set to parsed arguments. + + The pid and child_fd of this object get set by this method. + Note that it is difficult for this method to fail. + You cannot detect if the child process cannot start. + So the only way you can tell if the child process started + or not is to try to read from the file descriptor. If you get + EOF immediately then it means that the child is already dead. + That may not necessarily be bad because you may haved spawned a child + that performs some task; creates no stdout output; and then dies. + """ + self.searcher = None + self.ignorecase = False + self.before = None + self.after = None + self.match = None + self.match_index = None + self.terminated = True + self.exitstatus = None + self.status = None # status returned by os.waitpid + self.flag_eof = False + self.flag_child_finished = False + self.pid = None + self.child_fd = -1 # initially closed + self.timeout = timeout + self.delimiter = EOF + self.cwd = cwd + self.env = env + self.maxread = maxread # max bytes to read at one time into buffer + self.delaybeforesend = 0.05 # Sets sleep time used just before sending data to child. Time in seconds. + self.delayafterterminate = 0.1 # Sets delay in terminate() method to allow kernel time to update process status. Time in seconds. + self.flag_child_finished = False + self.buffer = '' # This is the read buffer. See maxread. + self.searchwindowsize = searchwindowsize # Anything before searchwindowsize point is preserved, but not searched. + + + # If command is an int type then it may represent a file descriptor. + if type(command) == type(0): + raise ExceptionPexpect ('Command is an int type. If this is a file descriptor then maybe you want to use fdpexpect.fdspawn which takes an existing file descriptor instead of a command string.') + + if type (args) != type([]): + raise TypeError ('The argument, args, must be a list.') + + if args == []: + self.args = split_command_line(command) + self.command = self.args[0] + else: + self.args = args[:] # work with a copy + self.args.insert (0, command) + self.command = command + + command_with_path = shutil.which(self.command) + if command_with_path is None: + raise ExceptionPexpect ('The command was not found or was not executable: %s.' % self.command) + self.command = command_with_path + self.args[0] = self.command + + self.name = '<' + ' '.join (self.args) + '>' + + if self.cwd is not None: + os.chdir(self.cwd) + + + if self.cwd is not None: + # Restore the original working dir + os.chdir(self.ocwd) + + self.terminated = False + self.closed = False + + self.child_fd = self.startChild(self.args, self.env) + self.connect_to_child('localhost', 4321) + + def __del__(self): + """This makes sure that no system resources are left open. Python only + garbage collects Python objects, not the child console.""" + + try: + self.terminate() + except: + pass + + def __str__(self): + + """This returns a human-readable string that represents the state of + the object. """ + + s = [] + s.append(repr(self)) + s.append('command: ' + str(self.command)) + s.append('args: ' + str(self.args)) + s.append('searcher: ' + str(self.searcher)) + s.append('buffer (last 100 chars): ' + str(self.buffer)[-100:]) + s.append('before (last 100 chars): ' + str(self.before)[-100:]) + s.append('after: ' + str(self.after)) + s.append('match: ' + str(self.match)) + s.append('match_index: ' + str(self.match_index)) + s.append('exitstatus: ' + str(self.exitstatus)) + s.append('flag_eof: ' + str(self.flag_eof)) + s.append('pid: ' + str(self.pid)) + s.append('child_fd: ' + str(self.child_fd)) + s.append('closed: ' + str(self.closed)) + s.append('timeout: ' + str(self.timeout)) + s.append('delimiter: ' + str(self.delimiter)) + s.append('maxread: ' + str(self.maxread)) + s.append('ignorecase: ' + str(self.ignorecase)) + s.append('searchwindowsize: ' + str(self.searchwindowsize)) + s.append('delaybeforesend: ' + str(self.delaybeforesend)) + s.append('delayafterterminate: ' + str(self.delayafterterminate)) + return '\n'.join(s) + + def fileno (self): # File-like object. + """There is no child fd.""" + + return 0 + + def terminate(self, force=False): + """Terminate the child. Force not used. """ + + if not self.isalive(): + return True + + win32api.TerminateProcess(self.conproc, 1) + time.sleep(self.delayafterterminate) + if not self.isalive(): + return True + + return False + + def close(self, force=True): # File-like object. + """ Closes the child console.""" + + self.closed = self.terminate(force) + if not self.closed: + raise ExceptionPexpect ('close() could not terminate the child using terminate()') + self.closed = True + + def read (self, size = -1): # File-like object. + """This reads at most "size" bytes from the file (less if the read hits + EOF before obtaining size bytes). If the size argument is negative or + omitted, read all data until EOF is reached. The bytes are returned as + a string object. An empty string is returned when EOF is encountered + immediately. """ + + if size == 0: + return '' + if size < 0: + self.expect (self.delimiter) # delimiter default is EOF + return self.before + + # I could have done this more directly by not using expect(), but + # I deliberately decided to couple read() to expect() so that + # I would catch any bugs early and ensure consistant behavior. + # It's a little less efficient, but there is less for me to + # worry about if I have to later modify read() or expect(). + # Note, it's OK if size==-1 in the regex. That just means it + # will never match anything in which case we stop only on EOF. + cre = re.compile('.{%d}' % size, re.DOTALL) + index = self.expect ([cre, self.delimiter]) # delimiter default is EOF + if index == 0: + return self.after ### self.before should be ''. Should I assert this? + return self.before + + def readline (self, size = -1): # File-like object. + + """This reads and returns one entire line. A trailing newline is kept + in the string, but may be absent when a file ends with an incomplete + line. Note: This readline() looks for a \\r\\n pair even on UNIX + because this is what the pseudo tty device returns. So contrary to what + you may expect you will receive the newline as \\r\\n. An empty string + is returned when EOF is hit immediately. Currently, the size argument is + mostly ignored, so this behavior is not standard for a file-like + object. If size is 0 then an empty string is returned. """ + + if size == 0: + return '' + index = self.expect (['\r\n', self.delimiter]) # delimiter default is EOF + if index == 0: + return self.before + '\r\n' + else: + return self.before + + def __iter__ (self): # File-like object. + + """This is to support iterators over a file-like object. + """ + + return self + + def read_nonblocking (self, size = 1): + """This reads at most size characters from the child application. If + the end of file is read then an EOF exception will be raised. + + This is not effected by the 'size' parameter, so if you call + read_nonblocking(size=100, timeout=30) and only one character is + available right away then one character will be returned immediately. + It will not wait for 30 seconds for another 99 characters to come in. + + This is a wrapper around Wtty.read(). """ + + if self.closed: + raise ValueError ('I/O operation on closed file in read_nonblocking().') + + try: + # The real child and it's console are two different process. The console dies 0.1 sec + # later to be able to read the child's last output (before EOF). So here we check + # isalive() (which checks the real child.) and try a last read on the console. To catch + # the last output. + # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. + if self.flag_child_finished: + raise EOF('self.flag_child_finished') + if not self.isalive(): + self.flag_child_finished = True + + s = self.sock.recv(size) + except EOF: + self.flag_eof = True + raise + + return s.decode() + + def __next__ (self): # File-like object. + + """This is to support iterators over a file-like object. + """ + + result = self.readline() + if self.after == self.delimiter: + raise StopIteration + return result + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.terminate() + + def readlines (self, sizehint = -1): # File-like object. + + """This reads until EOF using readline() and returns a list containing + the lines thus read. The optional "sizehint" argument is ignored. """ + + lines = [] + while True: + line = self.readline() + if not line: + break + lines.append(line) + return lines + + def isatty(self): # File-like object. + """The child is always created with a console.""" + + return True + + + def pipe_client(self, conpid): + pipe_name = 'wexpect_{}'.format(conpid) + pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) + print('Trying to connect to pipe: {}'.format(pipe_full_path)) + quit = False + + while not quit: + try: + handle = win32file.CreateFile( + pipe_full_path, + win32file.GENERIC_READ | win32file.GENERIC_WRITE, + 0, + None, + win32file.OPEN_EXISTING, + 0, + None + ) + print("pipe found!") + res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) + if res == 0: + print(f"SetNamedPipeHandleState return code: {res}") + while True: + resp = win32file.ReadFile(handle, 64*1024) + print(f"message: {resp}") + win32file.WriteFile(handle, b'back') + except pywintypes.error as e: + if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 + print("no pipe, trying again in a bit later") + time.sleep(0.2) + elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 + print("broken pipe, bye bye") + quit = True + elif e.args[0] == winerror.ERROR_NO_DATA: + '''232 (0xE8) + The pipe is being closed. + ''' + print("The pipe is being closed.") + quit = True + else: + raise + + + def connect_to_child(self, host, port): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((host, port)) + + + def startChild(self, args, env): + si = win32process.GetStartupInfo() + si.dwFlags = win32process.STARTF_USESHOWWINDOW + si.wShowWindow = win32con.SW_HIDE + + dirname = os.path.dirname(sys.executable + if getattr(sys, 'frozen', False) else + os.path.abspath(__file__)) + spath = [os.path.dirname(dirname)] + pyargs = ['-c'] + if getattr(sys, 'frozen', False): + # If we are running 'frozen', add library.zip and lib\library.zip + # to sys.path + # py2exe: Needs appropriate 'zipfile' option in setup script and + # 'bundle_files' 3 + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + if os.path.isdir(os.path.join(dirname, 'lib')): + dirname = os.path.join(dirname, 'lib') + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + pyargs.insert(0, '-S') # skip 'import site' + + + pid = win32process.GetCurrentProcessId() + + commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') + if getattr(sys, 'frozen', False) else + os.path.join(os.path.dirname(sys.executable), 'python.exe'), + ' '.join(pyargs), + "import sys;" + f"sys.path = {spath} + sys.path;" + "import wexpect;" + "import time;" + f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port=4321);" + ) + + print(commandLine) + + self.conproc, _, conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, + win32process.CREATE_NEW_CONSOLE, None, None, si) + + + def isalive(self, console=True): + """True if the child is still alive, false otherwise""" + + if console: + return win32process.GetExitCodeProcess(self.conproc) == win32con.STILL_ACTIVE + else: + return win32process.GetExitCodeProcess(self.__childProcess) == win32con.STILL_ACTIVE + + + def write(self, s): # File-like object. + + """This is similar to send() except that there is no return value. + """ + + self.send(s) + + def writelines (self, sequence): # File-like object. + + """This calls write() for each element in the sequence. The sequence + can be any iterable object producing strings, typically a list of + strings. This does not add line separators There is no return value. + """ + + for s in sequence: + self.write(s) + + def sendline(self, s=''): + + """This is like send(), but it adds a line feed (os.linesep). This + returns the number of bytes written. """ + + n = self.send(s) + n = n + self.send(b'\r\n') + return n + + def sendeof(self): + + """This sends an EOF to the child. This sends a character which causes + the pending parent output buffer to be sent to the waiting child + program without waiting for end-of-line. If it is the first character + of the line, the read() in the user program returns 0, which signifies + end-of-file. This means to work as expected a sendeof() has to be + called at the beginning of a line. This method does not send a newline. + It is the responsibility of the caller to ensure the eof is sent at the + beginning of a line. """ + + # platform does not define VEOF so assume CTRL-D + char = chr(4) + self.send(char) + + def send(self, s): + """This sends a string to the child process. This returns the number of + bytes written. If a log file was set then the data is also written to + the log. """ + + time.sleep(self.delaybeforesend) + self.sock.sendall(s) + return len(s) + + + def compile_pattern_list(self, patterns): + + """This compiles a pattern-string or a list of pattern-strings. + Patterns must be a StringType, EOF, TIMEOUT, SRE_Pattern, or a list of + those. Patterns may also be None which results in an empty list (you + might do this if waiting for an EOF or TIMEOUT condition without + expecting any pattern). + + This is used by expect() when calling expect_list(). Thus expect() is + nothing more than:: + + cpl = self.compile_pattern_list(pl) + return self.expect_list(cpl, timeout) + + If you are using expect() within a loop it may be more + efficient to compile the patterns first and then call expect_list(). + This avoid calls in a loop to compile_pattern_list():: + + cpl = self.compile_pattern_list(my_pattern) + while some_condition: + ... + i = self.expect_list(clp, timeout) + ... + """ + + if patterns is None: + return [] + if type(patterns) is not list: + patterns = [patterns] + + compile_flags = re.DOTALL # Allow dot to match \n + if self.ignorecase: + compile_flags = compile_flags | re.IGNORECASE + compiled_pattern_list = [] + for p in patterns: + if type(p) in (str,): + compiled_pattern_list.append(re.compile(p, compile_flags)) + elif p is EOF: + compiled_pattern_list.append(EOF) + elif p is TIMEOUT: + compiled_pattern_list.append(TIMEOUT) + elif type(p) is type(re.compile('')): + compiled_pattern_list.append(p) + else: + raise TypeError ('Argument must be one of StringTypes, EOF, TIMEOUT, SRE_Pattern, or a list of those type. %s' % str(type(p))) + + return compiled_pattern_list + + def expect(self, pattern, timeout = -1, searchwindowsize=None): + + """This seeks through the stream until a pattern is matched. The + pattern is overloaded and may take several types. The pattern can be a + StringType, EOF, a compiled re, or a list of any of those types. + Strings will be compiled to re types. This returns the index into the + pattern list. If the pattern was not a list this returns index 0 on a + successful match. This may raise exceptions for EOF or TIMEOUT. To + avoid the EOF or TIMEOUT exceptions add EOF or TIMEOUT to the pattern + list. That will cause expect to match an EOF or TIMEOUT condition + instead of raising an exception. + + If you pass a list of patterns and more than one matches, the first match + in the stream is chosen. If more than one pattern matches at that point, + the leftmost in the pattern list is chosen. For example:: + + # the input is 'foobar' + index = p.expect (['bar', 'foo', 'foobar']) + # returns 1 ('foo') even though 'foobar' is a "better" match + + Please note, however, that buffering can affect this behavior, since + input arrives in unpredictable chunks. For example:: + + # the input is 'foobar' + index = p.expect (['foobar', 'foo']) + # returns 0 ('foobar') if all input is available at once, + # but returs 1 ('foo') if parts of the final 'bar' arrive late + + After a match is found the instance attributes 'before', 'after' and + 'match' will be set. You can see all the data read before the match in + 'before'. You can see the data that was matched in 'after'. The + re.MatchObject used in the re match will be in 'match'. If an error + occurred then 'before' will be set to all the data read so far and + 'after' and 'match' will be None. + + If timeout is -1 then timeout will be set to the self.timeout value. + + A list entry may be EOF or TIMEOUT instead of a string. This will + catch these exceptions and return the index of the list entry instead + of raising the exception. The attribute 'after' will be set to the + exception type. The attribute 'match' will be None. This allows you to + write code like this:: + + index = p.expect (['good', 'bad', wexpect.EOF, wexpect.TIMEOUT]) + if index == 0: + do_something() + elif index == 1: + do_something_else() + elif index == 2: + do_some_other_thing() + elif index == 3: + do_something_completely_different() + + instead of code like this:: + + try: + index = p.expect (['good', 'bad']) + if index == 0: + do_something() + elif index == 1: + do_something_else() + except EOF: + do_some_other_thing() + except TIMEOUT: + do_something_completely_different() + + These two forms are equivalent. It all depends on what you want. You + can also just expect the EOF if you are waiting for all output of a + child to finish. For example:: + + p = wexpect.spawn('/bin/ls') + p.expect (wexpect.EOF) + print p.before + + If you are trying to optimize for speed then see expect_list(). + """ + + compiled_pattern_list = self.compile_pattern_list(pattern) + return self.expect_list(compiled_pattern_list, timeout, searchwindowsize) + + def expect_list(self, pattern_list, timeout = -1, searchwindowsize = -1): + + """This takes a list of compiled regular expressions and returns the + index into the pattern_list that matched the child output. The list may + also contain EOF or TIMEOUT (which are not compiled regular + expressions). This method is similar to the expect() method except that + expect_list() does not recompile the pattern list on every call. This + may help if you are trying to optimize for speed, otherwise just use + the expect() method. This is called by expect(). If timeout==-1 then + the self.timeout value is used. If searchwindowsize==-1 then the + self.searchwindowsize value is used. """ + + return self.expect_loop(searcher_re(pattern_list), timeout, searchwindowsize) + + def expect_exact(self, pattern_list, timeout = -1, searchwindowsize = -1): + + """This is similar to expect(), but uses plain string matching instead + of compiled regular expressions in 'pattern_list'. The 'pattern_list' + may be a string; a list or other sequence of strings; or TIMEOUT and + EOF. + + This call might be faster than expect() for two reasons: string + searching is faster than RE matching and it is possible to limit the + search to just the end of the input buffer. + + This method is also useful when you don't want to have to worry about + escaping regular expression characters that you want to match.""" + + if not isinstance(pattern_list, list): + pattern_list = [pattern_list] + + for p in pattern_list: + if type(p) not in (str,) and p not in (TIMEOUT, EOF): + raise TypeError ('Argument must be one of StringTypes, EOF, TIMEOUT, or a list of those type. %s' % str(type(p))) + + return self.expect_loop(searcher_string(pattern_list), timeout, searchwindowsize) + + def expect_loop(self, searcher, timeout = -1, searchwindowsize = -1): + + """This is the common loop used inside expect. The 'searcher' should be + an instance of searcher_re or searcher_string, which describes how and what + to search for in the input. + + See expect() for other arguments, return value and exceptions. """ + + self.searcher = searcher + + if timeout == -1: + timeout = self.timeout + if timeout is not None: + end_time = time.time() + timeout + if searchwindowsize == -1: + searchwindowsize = self.searchwindowsize + + try: + incoming = self.buffer + freshlen = len(incoming) + while True: # Keep reading until exception or return. + index = searcher.search(incoming, freshlen, searchwindowsize) + if index >= 0: + self.buffer = incoming[searcher.end : ] + self.before = incoming[ : searcher.start] + self.after = incoming[searcher.start : searcher.end] + self.match = searcher.match + self.match_index = index + return self.match_index + # No match at this point + if timeout is not None and end_time < time.time(): + raise TIMEOUT ('Timeout exceeded in expect_any().') + # Still have time left, so read more data + c = self.read_nonblocking(self.maxread) + freshlen = len(c) + time.sleep (0.01) + incoming += c + except EOF as e: + self.buffer = '' + self.before = incoming + self.after = EOF + index = searcher.eof_index + if index >= 0: + self.match = EOF + self.match_index = index + return self.match_index + else: + self.match = None + self.match_index = None + raise EOF (str(e) + '\n' + str(self)) + except TIMEOUT as e: + self.buffer = incoming + self.before = incoming + self.after = TIMEOUT + index = searcher.timeout_index + if index >= 0: + self.match = TIMEOUT + self.match_index = index + return self.match_index + else: + self.match = None + self.match_index = None + raise TIMEOUT (str(e) + '\n' + str(self)) + except: + self.before = incoming + self.after = None + self.match = None + self.match_index = None + raise + + +class searcher_re (object): + + """This is regular expression string search helper for the + spawn.expect_any() method. + + Attributes: + + eof_index - index of EOF, or -1 + timeout_index - index of TIMEOUT, or -1 + + After a successful match by the search() method the following attributes + are available: + + start - index into the buffer, first byte of match + end - index into the buffer, first byte after match + match - the re.match object returned by a succesful re.search + + """ + + def __init__(self, patterns): + + """This creates an instance that searches for 'patterns' Where + 'patterns' may be a list or other sequence of compiled regular + expressions, or the EOF or TIMEOUT types.""" + + self.eof_index = -1 + self.timeout_index = -1 + self._searches = [] + for n, s in zip(list(range(len(patterns))), patterns): + if s is EOF: + self.eof_index = n + continue + if s is TIMEOUT: + self.timeout_index = n + continue + self._searches.append((n, s)) + + def __str__(self): + + """This returns a human-readable string that represents the state of + the object.""" + + ss = [ (n,' %d: re.compile("%s")' % (n,str(s.pattern))) for n,s in self._searches] + ss.append((-1,'searcher_re:')) + if self.eof_index >= 0: + ss.append ((self.eof_index,' %d: EOF' % self.eof_index)) + if self.timeout_index >= 0: + ss.append ((self.timeout_index,' %d: TIMEOUT' % self.timeout_index)) + ss.sort() + ss = list(zip(*ss))[1] + return '\n'.join(ss) + + def search(self, buffer, freshlen, searchwindowsize=None): + + """This searches 'buffer' for the first occurence of one of the regular + expressions. 'freshlen' must indicate the number of bytes at the end of + 'buffer' which have not been searched before. + + See class spawn for the 'searchwindowsize' argument. + + If there is a match this returns the index of that string, and sets + 'start', 'end' and 'match'. Otherwise, returns -1.""" + + absurd_match = len(buffer) + first_match = absurd_match + # 'freshlen' doesn't help here -- we cannot predict the + # length of a match, and the re module provides no help. + if searchwindowsize is None: + searchstart = 0 + else: + searchstart = max(0, len(buffer)-searchwindowsize) + for index, s in self._searches: + match = s.search(buffer, searchstart) + if match is None: + continue + n = match.start() + if n < first_match: + first_match = n + the_match = match + best_index = index + if first_match == absurd_match: + return -1 + self.start = first_match + self.match = the_match + self.end = self.match.end() + return best_index + + + +def main(): + try: + p = Spawn('cmd') + + p.sendline(b'ls') + time.sleep(.5) + data = p.expect('>') + print(data) + print(p.before) + data = p.expect('>') + print(data) + print(p.before) + + except: + traceback.print_exc() + finally: + p.terminate() + + +if __name__ == '__main__': + main() + \ No newline at end of file diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py new file mode 100644 index 0000000..5c0e93e --- /dev/null +++ b/wexpect/wexpect_util.py @@ -0,0 +1,105 @@ +"""Wexpect is a Windows variant of pexpect https://pexpect.readthedocs.io. + +Wexpect is a Python module for spawning child applications and controlling +them automatically. + +wexpect util contains small functions, and classes, which are used in multiple classes. +The command line argument parsers, and the Exceptions placed here. + +Credits: Noah Spurrier, Richard Holden, Marco Molteni, Kimberley Burchett, +Robert Stone, Hartmut Goebel, Chad Schroeder, Erick Tryzelaar, Dave Kirby, Ids +vander Molen, George Todd, Noel Taylor, Nicolas D. Cesar, Alexander Gattin, +Geoffrey Marshall, Francisco Lourenco, Glen Mabey, Karthik Gurusamy, Fernando +Perez, Corey Minyard, Jon Cohen, Guillaume Chazarain, Andrew Ryan, Nick +Craig-Wood, Andrew Stone, Jorgen Grahn, Benedek Racz + +Free, open source, and all that good stuff. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Wexpect Copyright (c) 2019 Benedek Racz + +""" + +import re +import ctypes +import traceback +import sys + +def split_command_line(command_line): + '''https://stackoverflow.com/a/35900070/2506522 + ''' + + nargs = ctypes.c_int() + ctypes.windll.shell32.CommandLineToArgvW.restype = ctypes.POINTER(ctypes.c_wchar_p) + lpargs = ctypes.windll.shell32.CommandLineToArgvW(command_line, ctypes.byref(nargs)) + args = [lpargs[i] for i in range(nargs.value)] + if ctypes.windll.kernel32.LocalFree(lpargs): + raise AssertionError + return args + +def join_args(args): + """Joins arguments into a command line. It quotes all arguments that contain + spaces or any of the characters ^!$%&()[]{}=;'+,`~""" + commandline = [] + for arg in args: + if re.search('[\^!$%&()[\]{}=;\'+,`~\s]', arg): + arg = '"%s"' % arg + commandline.append(arg) + return ' '.join(commandline) + + +class ExceptionPexpect(Exception): + """Base class for all exceptions raised by this module. + """ + + def __init__(self, value): + + self.value = value + + def __str__(self): + + return str(self.value) + + def get_trace(self): + """This returns an abbreviated stack trace with lines that only concern + the caller. In other words, the stack trace inside the Wexpect module + is not included. """ + + tblist = traceback.extract_tb(sys.exc_info()[2]) + tblist = [item for item in tblist if self.__filter_not_wexpect(item)] + tblist = traceback.format_list(tblist) + return ''.join(tblist) + + def __filter_not_wexpect(self, trace_list_item): + """This returns True if list item 0 the string 'wexpect.py' in it. """ + + if trace_list_item[0].find('wexpect.py') == -1: + return True + else: + return False + + +class EOF(ExceptionPexpect): + """Raised when EOF is read from a child. This usually means the child has exited. + The user can wait to EOF, which means he waits the end of the execution of the child process.""" + +class TIMEOUT(ExceptionPexpect): + """Raised when a read time exceeds the timeout. """ + From ba20b3f0d8727a2ad75e0a4d22c331fde6e9eae2 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Mon, 20 Jan 2020 13:27:40 +0100 Subject: [PATCH 03/22] [ADD] run function --- wexpect/__init__.py | 3 +- wexpect/spawn.py | 123 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 119 insertions(+), 7 deletions(-) diff --git a/wexpect/__init__.py b/wexpect/__init__.py index 02d6f64..f7b46ea 100644 --- a/wexpect/__init__.py +++ b/wexpect/__init__.py @@ -11,6 +11,7 @@ from .console_reader import ConsoleReaderPipe from .spawn import Spawn from .spawn import Spawn as spawn +from .spawn import run __all__ = ['split_command_line', 'join_args', 'ExceptionPexpect', 'EOF', 'TIMEOUT', - 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'Spawn'] + 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'Spawn', 'run'] diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 6e96e34..a1db127 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -70,6 +70,7 @@ import os import shutil import re import traceback +import types import pywintypes import win32process @@ -80,10 +81,117 @@ import winerror import win32pipe import socket -from wexpect_util import ExceptionPexpect -from wexpect_util import EOF -from wexpect_util import TIMEOUT -from wexpect_util import split_command_line +from .wexpect_util import ExceptionPexpect +from .wexpect_util import EOF +from .wexpect_util import TIMEOUT +from .wexpect_util import split_command_line + + +def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None, logfile=None, cwd=None, env=None): + + """ + This function runs the given command; waits for it to finish; then + returns all output as a string. STDERR is included in output. If the full + path to the command is not given then the path is searched. + + Note that lines are terminated by CR/LF (\\r\\n) combination even on + UNIX-like systems because this is the standard for pseudo ttys. If you set + 'withexitstatus' to true, then run will return a tuple of (command_output, + exitstatus). If 'withexitstatus' is false then this returns just + command_output. + + The run() function can often be used instead of creating a spawn instance. + For example, the following code uses spawn:: + + child = spawn('scp foo myname@host.example.com:.') + child.expect ('(?i)password') + child.sendline (mypassword) + + The previous code can be replace with the following:: + + Examples + ======== + + Start the apache daemon on the local machine:: + + run ("/usr/local/apache/bin/apachectl start") + + Check in a file using SVN:: + + run ("svn ci -m 'automatic commit' my_file.py") + + Run a command and capture exit status:: + + (command_output, exitstatus) = run ('ls -l /bin', withexitstatus=1) + + Tricky Examples + =============== + + The following will run SSH and execute 'ls -l' on the remote machine. The + password 'secret' will be sent if the '(?i)password' pattern is ever seen:: + + run ("ssh username@machine.example.com 'ls -l'", events={'(?i)password':'secret\\n'}) + + This will start mencoder to rip a video from DVD. This will also display + progress ticks every 5 seconds as it runs. For example:: + + The 'events' argument should be a dictionary of patterns and responses. + Whenever one of the patterns is seen in the command out run() will send the + associated response string. Note that you should put newlines in your + string if Enter is necessary. The responses may also contain callback + functions. Any callback is function that takes a dictionary as an argument. + The dictionary contains all the locals from the run() function, so you can + access the child spawn object or any other variable defined in run() + (event_count, child, and extra_args are the most useful). A callback may + return True to stop the current run process otherwise run() continues until + the next event. A callback may also return a string which will be sent to + the child. 'extra_args' is not used by directly run(). It provides a way to + pass data to a callback function through run() through the locals + dictionary passed to a callback. """ + + if timeout == -1: + child = Spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env) + else: + child = Spawn(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env) + if events is not None: + patterns = list(events.keys()) + responses = list(events.values()) + else: + patterns=None # We assume that EOF or TIMEOUT will save us. + responses=None + child_result_list = [] + event_count = 0 + while 1: + try: + index = child.expect (patterns) + if type(child.after) in (str,): + child_result_list.append(child.before + child.after) + else: # child.after may have been a TIMEOUT or EOF, so don't cat those. + child_result_list.append(child.before) + if type(responses[index]) in (str,): + child.send(responses[index]) + elif type(responses[index]) is types.FunctionType: + callback_result = responses[index](locals()) + sys.stdout.flush() + if type(callback_result) in (str,): + child.send(callback_result) + elif callback_result: + break + else: + raise TypeError ('The callback must be a string or function type.') + event_count = event_count + 1 + except TIMEOUT: + child_result_list.append(child.before) + break + except EOF: + child_result_list.append(child.before) + break + child_result = ''.join(child_result_list) + if withexitstatus: + child.close() + return (child_result, child.exitstatus) + else: + return child_result class Spawn: @@ -171,6 +279,7 @@ class Spawn: try: self.terminate() + self.disconnect_from_child() except: pass @@ -393,6 +502,10 @@ class Spawn: def connect_to_child(self, host, port): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect((host, port)) + + def disconnect_from_child(self): + if self.sock: + self.sock.close() def startChild(self, args, env): @@ -433,8 +546,6 @@ class Spawn: "import time;" f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port=4321);" ) - - print(commandLine) self.conproc, _, conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, win32process.CREATE_NEW_CONSOLE, None, None, si) From 46a206ae3046ee02f03eb07e28c980d0b277c833 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Tue, 21 Jan 2020 09:36:46 +0100 Subject: [PATCH 04/22] [ADD] SpawnPipe, and SpawnSocket classes --- tests/test_command_list_split.py | 8 +- wexpect/__init__.py | 7 +- wexpect/console_reader.py | 76 +++---- wexpect/spawn.py | 326 ++++++++++++++++++------------- wexpect/wexpect_util.py | 65 ++++-- 5 files changed, 290 insertions(+), 192 deletions(-) diff --git a/tests/test_command_list_split.py b/tests/test_command_list_split.py index 9ae5396..de8b9b2 100644 --- a/tests/test_command_list_split.py +++ b/tests/test_command_list_split.py @@ -22,8 +22,8 @@ import wexpect import unittest from . import PexpectTestCase -class SplitCommandLineTestCase(PexpectTestCase.PexpectTestCase): - def testSplitSizes(self): +class TestCaseSplitCommandLine(PexpectTestCase.PexpectTestCase): + def test_split_sizes(self): self.assertEqual(len(wexpect.split_command_line(r'')), 0) self.assertEqual(len(wexpect.split_command_line(r'one')), 1) self.assertEqual(len(wexpect.split_command_line(r'one two')), 2) @@ -34,7 +34,7 @@ class SplitCommandLineTestCase(PexpectTestCase.PexpectTestCase): self.assertEqual(len(wexpect.split_command_line(r'one\"one')), 1) self.assertEqual(len(wexpect.split_command_line(r"This^' is a^'^ test")), 3) - def testJoinArgs(self): + def test_join_args(self): cmd = 'foo bar "b a z"' cmd2 = wexpect.join_args(wexpect.split_command_line(cmd)) self.assertEqual(cmd2, cmd) @@ -46,4 +46,4 @@ class SplitCommandLineTestCase(PexpectTestCase.PexpectTestCase): if __name__ == '__main__': unittest.main() -suite = unittest.makeSuite(SplitCommandLineTestCase,'test') +suite = unittest.makeSuite(TestCaseSplitCommandLine,'test') diff --git a/wexpect/__init__.py b/wexpect/__init__.py index f7b46ea..27ffa56 100644 --- a/wexpect/__init__.py +++ b/wexpect/__init__.py @@ -9,9 +9,10 @@ from .wexpect_util import TIMEOUT from .console_reader import ConsoleReaderSocket from .console_reader import ConsoleReaderPipe -from .spawn import Spawn -from .spawn import Spawn as spawn +from .spawn import SpawnSocket +from .spawn import SpawnPipe +from .spawn import SpawnSocket as spawn from .spawn import run __all__ = ['split_command_line', 'join_args', 'ExceptionPexpect', 'EOF', 'TIMEOUT', - 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'Spawn', 'run'] + 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'SpawnSocket', 'SpawnPipe', 'run'] diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index cf9cb39..e5b2f05 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -72,16 +72,24 @@ except: # pragma: no cover # console manipulation. # logger = logging.getLogger('wexpect') -os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' -try: - logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] - logger.setLevel(logger_level) - fh = logging.FileHandler('wexpect.log', 'w', 'utf-8') - formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') - fh.setFormatter(formatter) - logger.addHandler(fh) -except KeyError: - logger.setLevel(logging.ERROR) + +def init_logger(): + logger = logging.getLogger('wexpect') + os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' + try: + logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] + try: + logger_filename = os.environ['WEXPECT_LOGGER_FILENAME'] + except KeyError: + pid = os.getpid() + logger_filename = f'wexpect_{pid}' + logger.setLevel(logger_level) + fh = logging.FileHandler(f'{logger_filename}.log', 'w', 'utf-8') + formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) + except KeyError: + logger.setLevel(logging.ERROR) class ConsoleReaderBase: """Consol class (aka. client-side python class) for the child. @@ -109,7 +117,9 @@ class ConsoleReaderBase: self.consin = None self.consout = None self.local_echo = True + self.pid = os.getpid() + init_logger() logger.info("ConsoleReader started") if cp: @@ -129,13 +139,8 @@ class ConsoleReaderBase: self.__childProcess, _, childPid, self.__tid = win32process.CreateProcess(None, path, None, None, False, 0, None, None, si) - print('123') - print('456') - print('789') - except Exception as e: logger.info(e) - time.sleep(.1) return time.sleep(.2) @@ -162,10 +167,6 @@ class ConsoleReaderBase: """ if e.args[0] != winerror.ERROR_ACCESS_DENIED: logger.info(e) - - time.sleep(.1) - self.send_to_host(self.readConsoleToCursor()) - time.sleep(.1) return if cursorPos.Y > maxconsoleY and not paused: @@ -183,6 +184,9 @@ class ConsoleReaderBase: logger.error(traceback.format_exc()) time.sleep(.1) finally: + time.sleep(.1) + self.send_to_host(self.readConsoleToCursor()) + time.sleep(1) self.close_connection() def write(self, s): @@ -405,19 +409,22 @@ class ConsoleReaderSocket(ConsoleReaderBase): def create_connection(self, **kwargs): - - self.port = kwargs['port'] - # Create a TCP/IP socket - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_address = ('localhost', self.port) - self.sock.bind(server_address) - logger.info(f'Socket started at port: {self.port}') - - # Listen for incoming connections - self.sock.listen(1) - self.connection, client_address = self.sock.accept() - self.connection.settimeout(.2) - logger.info(f'Client connected: {client_address}') + try: + self.port = kwargs['port'] + # Create a TCP/IP socket + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_address = ('localhost', self.port) + self.sock.bind(server_address) + logger.info(f'Socket started at port: {self.port}') + + # Listen for incoming connections + self.sock.listen(1) + self.connection, client_address = self.sock.accept() + self.connection.settimeout(.2) + logger.info(f'Client connected: {client_address}') + except: + logger.error(f"Port: {self.port}") + raise def close_connection(self): if self.connection: @@ -449,9 +456,8 @@ class ConsoleReaderSocket(ConsoleReaderBase): class ConsoleReaderPipe(ConsoleReaderBase): - def create_connection(self): - pid = win32process.GetCurrentProcessId() - pipe_name = 'wexpect_{}'.format(pid) + def create_connection(self, **kwargs): + pipe_name = 'wexpect_{}'.format(self.pid) pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) logger.info('Start pipe server: %s', pipe_full_path) self.pipe = win32pipe.CreateNamedPipe( diff --git a/wexpect/spawn.py b/wexpect/spawn.py index a1db127..15f62a7 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -150,9 +150,9 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None dictionary passed to a callback. """ if timeout == -1: - child = Spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env) + child = SpawnSocket(command, maxread=2000, logfile=logfile, cwd=cwd, env=env) else: - child = Spawn(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env) + child = SpawnSocket(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env) if events is not None: patterns = list(events.keys()) responses = list(events.values()) @@ -194,7 +194,7 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None return child_result -class Spawn: +class SpawnBase: def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, logfile=None, cwd=None, env=None, codepage=None, echo=True): """This starts the given command in a child process. This does all the @@ -271,7 +271,7 @@ class Spawn: self.closed = False self.child_fd = self.startChild(self.args, self.env) - self.connect_to_child('localhost', 4321) + self.connect_to_child() def __del__(self): """This makes sure that no system resources are left open. Python only @@ -391,36 +391,9 @@ class Spawn: return self def read_nonblocking (self, size = 1): - """This reads at most size characters from the child application. If - the end of file is read then an EOF exception will be raised. - - This is not effected by the 'size' parameter, so if you call - read_nonblocking(size=100, timeout=30) and only one character is - available right away then one character will be returned immediately. - It will not wait for 30 seconds for another 99 characters to come in. - - This is a wrapper around Wtty.read(). """ - - if self.closed: - raise ValueError ('I/O operation on closed file in read_nonblocking().') - - try: - # The real child and it's console are two different process. The console dies 0.1 sec - # later to be able to read the child's last output (before EOF). So here we check - # isalive() (which checks the real child.) and try a last read on the console. To catch - # the last output. - # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. - if self.flag_child_finished: - raise EOF('self.flag_child_finished') - if not self.isalive(): - self.flag_child_finished = True - - s = self.sock.recv(size) - except EOF: - self.flag_eof = True - raise - - return s.decode() + """Virtual definition + """ + raise NotImplementedError def __next__ (self): # File-like object. @@ -455,101 +428,12 @@ class Spawn: """The child is always created with a console.""" return True - - - def pipe_client(self, conpid): - pipe_name = 'wexpect_{}'.format(conpid) - pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) - print('Trying to connect to pipe: {}'.format(pipe_full_path)) - quit = False - while not quit: - try: - handle = win32file.CreateFile( - pipe_full_path, - win32file.GENERIC_READ | win32file.GENERIC_WRITE, - 0, - None, - win32file.OPEN_EXISTING, - 0, - None - ) - print("pipe found!") - res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) - if res == 0: - print(f"SetNamedPipeHandleState return code: {res}") - while True: - resp = win32file.ReadFile(handle, 64*1024) - print(f"message: {resp}") - win32file.WriteFile(handle, b'back') - except pywintypes.error as e: - if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 - print("no pipe, trying again in a bit later") - time.sleep(0.2) - elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 - print("broken pipe, bye bye") - quit = True - elif e.args[0] == winerror.ERROR_NO_DATA: - '''232 (0xE8) - The pipe is being closed. - ''' - print("The pipe is being closed.") - quit = True - else: - raise - - - def connect_to_child(self, host, port): - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.connect((host, port)) - - def disconnect_from_child(self): - if self.sock: - self.sock.close() - - - def startChild(self, args, env): - si = win32process.GetStartupInfo() - si.dwFlags = win32process.STARTF_USESHOWWINDOW - si.wShowWindow = win32con.SW_HIDE - - dirname = os.path.dirname(sys.executable - if getattr(sys, 'frozen', False) else - os.path.abspath(__file__)) - spath = [os.path.dirname(dirname)] - pyargs = ['-c'] - if getattr(sys, 'frozen', False): - # If we are running 'frozen', add library.zip and lib\library.zip - # to sys.path - # py2exe: Needs appropriate 'zipfile' option in setup script and - # 'bundle_files' 3 - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - if os.path.isdir(os.path.join(dirname, 'lib')): - dirname = os.path.join(dirname, 'lib') - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - pyargs.insert(0, '-S') # skip 'import site' - - - pid = win32process.GetCurrentProcessId() - - commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') - if getattr(sys, 'frozen', False) else - os.path.join(os.path.dirname(sys.executable), 'python.exe'), - ' '.join(pyargs), - "import sys;" - f"sys.path = {spath} + sys.path;" - "import wexpect;" - "import time;" - f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port=4321);" - ) - - self.conproc, _, conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, - win32process.CREATE_NEW_CONSOLE, None, None, si) + def kill(self, sig): + """Sig == sigint for ctrl-c otherwise the child is terminated.""" + os.kill(self.conpid, sig) +# win32api.TerminateProcess(self.conproc, 1) def isalive(self, console=True): """True if the child is still alive, false otherwise""" @@ -600,16 +484,21 @@ class Spawn: # platform does not define VEOF so assume CTRL-D char = chr(4) self.send(char) - - def send(self, s): - """This sends a string to the child process. This returns the number of - bytes written. If a log file was set then the data is also written to - the log. """ - time.sleep(self.delaybeforesend) - self.sock.sendall(s) - return len(s) - + def send(self): + """Virtual definition + """ + raise NotImplementedError + + def connect_to_child(self): + """Virtual definition + """ + raise NotImplementedError + + def disconnect_from_child(self): + """Virtual definition + """ + raise NotImplementedError def compile_pattern_list(self, patterns): @@ -846,6 +735,167 @@ class Spawn: self.match_index = None raise +class SpawnPipe(SpawnBase): + + + + def pipe_client(self, conpid): + pipe_name = 'wexpect_{}'.format(conpid) + pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) + print('Trying to connect to pipe: {}'.format(pipe_full_path)) + quit = False + + while not quit: + try: + handle = win32file.CreateFile( + pipe_full_path, + win32file.GENERIC_READ | win32file.GENERIC_WRITE, + 0, + None, + win32file.OPEN_EXISTING, + 0, + None + ) + print("pipe found!") + res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) + if res == 0: + print(f"SetNamedPipeHandleState return code: {res}") + while True: + resp = win32file.ReadFile(handle, 64*1024) + print(f"message: {resp}") + win32file.WriteFile(handle, b'back') + except pywintypes.error as e: + if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 + print("no pipe, trying again in a bit later") + time.sleep(0.2) + elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 + print("broken pipe, bye bye") + quit = True + elif e.args[0] == winerror.ERROR_NO_DATA: + '''232 (0xE8) + The pipe is being closed. + ''' + print("The pipe is being closed.") + quit = True + else: + raise + + def send(self, s): + """This sends a string to the child process. This returns the number of + bytes written. If a log file was set then the data is also written to + the log. """ + if isinstance(s, str): + s = str.encode(s) + if self.delaybeforesend: + time.sleep(self.delaybeforesend) + self.sock.sendall(s) + return len(s) + + +class SpawnSocket(SpawnBase): + + def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, + logfile=None, cwd=None, env=None, codepage=None, echo=True, port=4321, host='localhost'): + self.port = port + self.host = host + super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, + searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo) + + + def send(self, s): + """This sends a string to the child process. This returns the number of + bytes written. If a log file was set then the data is also written to + the log. """ + if isinstance(s, str): + s = str.encode(s) + if self.delaybeforesend: + time.sleep(self.delaybeforesend) + self.sock.sendall(s) + return len(s) + + def connect_to_child(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((self.host, self.port)) + + def disconnect_from_child(self): + if self.sock: + self.sock.close() + + def read_nonblocking (self, size = 1): + """This reads at most size characters from the child application. If + the end of file is read then an EOF exception will be raised. + + This is not effected by the 'size' parameter, so if you call + read_nonblocking(size=100, timeout=30) and only one character is + available right away then one character will be returned immediately. + It will not wait for 30 seconds for another 99 characters to come in. + + This is a wrapper around Wtty.read(). """ + + if self.closed: + raise ValueError ('I/O operation on closed file in read_nonblocking().') + + try: + # The real child and it's console are two different process. The console dies 0.1 sec + # later to be able to read the child's last output (before EOF). So here we check + # isalive() (which checks the real child.) and try a last read on the console. To catch + # the last output. + # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. + if self.flag_child_finished: + raise EOF('self.flag_child_finished') + if not self.isalive(): + self.flag_child_finished = True + + s = self.sock.recv(size) + except EOF: + self.flag_eof = True + raise + + return s.decode() + + def startChild(self, args, env): + si = win32process.GetStartupInfo() + si.dwFlags = win32process.STARTF_USESHOWWINDOW + si.wShowWindow = win32con.SW_HIDE + + dirname = os.path.dirname(sys.executable + if getattr(sys, 'frozen', False) else + os.path.abspath(__file__)) + spath = [os.path.dirname(dirname)] + pyargs = ['-c'] + if getattr(sys, 'frozen', False): + # If we are running 'frozen', add library.zip and lib\library.zip + # to sys.path + # py2exe: Needs appropriate 'zipfile' option in setup script and + # 'bundle_files' 3 + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + if os.path.isdir(os.path.join(dirname, 'lib')): + dirname = os.path.join(dirname, 'lib') + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + pyargs.insert(0, '-S') # skip 'import site' + + + pid = win32process.GetCurrentProcessId() + + commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') + if getattr(sys, 'frozen', False) else + os.path.join(os.path.dirname(sys.executable), 'python.exe'), + ' '.join(pyargs), + "import sys;" + f"sys.path = {spath} + sys.path;" + "import wexpect;" + "import time;" + f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port});" + ) + + self.conproc, _, self.conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, + win32process.CREATE_NEW_CONSOLE, None, None, si) + + class searcher_re (object): @@ -938,7 +988,7 @@ class searcher_re (object): def main(): try: - p = Spawn('cmd') + p = SpawnSocket('cmd') p.sendline(b'ls') time.sleep(.5) @@ -957,4 +1007,4 @@ def main(): if __name__ == '__main__': main() - \ No newline at end of file + diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index 5c0e93e..ee17e8c 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -38,21 +38,62 @@ Wexpect Copyright (c) 2019 Benedek Racz """ import re -import ctypes import traceback import sys -def split_command_line(command_line): - '''https://stackoverflow.com/a/35900070/2506522 - ''' - - nargs = ctypes.c_int() - ctypes.windll.shell32.CommandLineToArgvW.restype = ctypes.POINTER(ctypes.c_wchar_p) - lpargs = ctypes.windll.shell32.CommandLineToArgvW(command_line, ctypes.byref(nargs)) - args = [lpargs[i] for i in range(nargs.value)] - if ctypes.windll.kernel32.LocalFree(lpargs): - raise AssertionError - return args +def split_command_line(command_line, escape_char = '^'): + """This splits a command line into a list of arguments. It splits arguments + on spaces, but handles embedded quotes, doublequotes, and escaped + characters. It's impossible to do this with a regular expression, so I + wrote a little state machine to parse the command line. """ + + arg_list = [] + arg = '' + + # Constants to name the states we can be in. + state_basic = 0 + state_esc = 1 + state_singlequote = 2 + state_doublequote = 3 + state_whitespace = 4 # The state of consuming whitespace between commands. + state = state_basic + + for c in command_line: + if state == state_basic or state == state_whitespace: + if c == escape_char: # Escape the next character + state = state_esc + elif c == r"'": # Handle single quote + state = state_singlequote + elif c == r'"': # Handle double quote + state = state_doublequote + elif c.isspace(): + # Add arg to arg_list if we aren't in the middle of whitespace. + if state == state_whitespace: + None # Do nothing. + else: + arg_list.append(arg) + arg = '' + state = state_whitespace + else: + arg = arg + c + state = state_basic + elif state == state_esc: + arg = arg + c + state = state_basic + elif state == state_singlequote: + if c == r"'": + state = state_basic + else: + arg = arg + c + elif state == state_doublequote: + if c == r'"': + state = state_basic + else: + arg = arg + c + + if arg != '': + arg_list.append(arg) + return arg_list def join_args(args): """Joins arguments into a command line. It quotes all arguments that contain From 3ea1346012889f2de9c16962cfc7b243a8cf8d45 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Tue, 21 Jan 2020 10:46:36 +0100 Subject: [PATCH 05/22] [FIX] logging, now all logs generated into .wlog dir; [ADD] searcher_string class --- wexpect/console_reader.py | 14 +++--- wexpect/spawn.py | 97 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index e5b2f05..cb15484 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -82,9 +82,11 @@ def init_logger(): logger_filename = os.environ['WEXPECT_LOGGER_FILENAME'] except KeyError: pid = os.getpid() - logger_filename = f'wexpect_{pid}' + logger_filename = f'./.wlog/wexpect_{pid}' logger.setLevel(logger_level) - fh = logging.FileHandler(f'{logger_filename}.log', 'w', 'utf-8') + logger_filename = f'{logger_filename}.log' + os.makedirs(os.path.dirname(logger_filename), exist_ok=True) + fh = logging.FileHandler(logger_filename, 'w', 'utf-8') formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) logger.addHandler(fh) @@ -142,10 +144,6 @@ class ConsoleReaderBase: except Exception as e: logger.info(e) return - - time.sleep(.2) - self.write('ls') - self.write(os.linesep) paused = False @@ -186,7 +184,7 @@ class ConsoleReaderBase: finally: time.sleep(.1) self.send_to_host(self.readConsoleToCursor()) - time.sleep(1) + time.sleep(.1) self.close_connection() def write(self, s): @@ -483,4 +481,4 @@ class ConsoleReaderPipe(ConsoleReaderBase): ret = resp[1] return ret - \ No newline at end of file + diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 15f62a7..d427aa3 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -983,6 +983,103 @@ class searcher_re (object): self.match = the_match self.end = self.match.end() return best_index + + +class searcher_string (object): + + """This is a plain string search helper for the spawn.expect_any() method. + + Attributes: + + eof_index - index of EOF, or -1 + timeout_index - index of TIMEOUT, or -1 + + After a successful match by the search() method the following attributes + are available: + + start - index into the buffer, first byte of match + end - index into the buffer, first byte after match + match - the matching string itself + """ + + def __init__(self, strings): + + """This creates an instance of searcher_string. This argument 'strings' + may be a list; a sequence of strings; or the EOF or TIMEOUT types. """ + + self.eof_index = -1 + self.timeout_index = -1 + self._strings = [] + for n, s in zip(list(range(len(strings))), strings): + if s is EOF: + self.eof_index = n + continue + if s is TIMEOUT: + self.timeout_index = n + continue + self._strings.append((n, s)) + + def __str__(self): + + """This returns a human-readable string that represents the state of + the object.""" + + ss = [ (ns[0],' %d: "%s"' % ns) for ns in self._strings ] + ss.append((-1,'searcher_string:')) + if self.eof_index >= 0: + ss.append ((self.eof_index,' %d: EOF' % self.eof_index)) + if self.timeout_index >= 0: + ss.append ((self.timeout_index,' %d: TIMEOUT' % self.timeout_index)) + ss.sort() + ss = list(zip(*ss))[1] + return '\n'.join(ss) + + def search(self, buffer, freshlen, searchwindowsize=None): + + """This searches 'buffer' for the first occurence of one of the search + strings. 'freshlen' must indicate the number of bytes at the end of + 'buffer' which have not been searched before. It helps to avoid + searching the same, possibly big, buffer over and over again. + + See class spawn for the 'searchwindowsize' argument. + + If there is a match this returns the index of that string, and sets + 'start', 'end' and 'match'. Otherwise, this returns -1. """ + + absurd_match = len(buffer) + first_match = absurd_match + + # 'freshlen' helps a lot here. Further optimizations could + # possibly include: + # + # using something like the Boyer-Moore Fast String Searching + # Algorithm; pre-compiling the search through a list of + # strings into something that can scan the input once to + # search for all N strings; realize that if we search for + # ['bar', 'baz'] and the input is '...foo' we need not bother + # rescanning until we've read three more bytes. + # + # Sadly, I don't know enough about this interesting topic. /grahn + + for index, s in self._strings: + if searchwindowsize is None: + # the match, if any, can only be in the fresh data, + # or at the very end of the old data + offset = -(freshlen+len(s)) + else: + # better obey searchwindowsize + offset = -searchwindowsize + n = buffer.find(s, offset) + if n >= 0 and n < first_match: + first_match = n + best_index, best_match = index, s + if first_match == absurd_match: + return -1 + self.match = best_match + self.start = first_match + self.end = self.start + len(self.match) + return best_index + From 4a7385e4a1eecb523c891d5279bb601a1c79f19a Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Tue, 21 Jan 2020 17:45:44 +0100 Subject: [PATCH 06/22] [FIX] many fixes around process termination; [ADD] pipe mode basic features added --- wexpect/console_reader.py | 94 ++++++++------ wexpect/spawn.py | 260 ++++++++++++++++++++++++++------------ 2 files changed, 230 insertions(+), 124 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index cb15484..3ac0398 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -87,11 +87,13 @@ def init_logger(): logger_filename = f'{logger_filename}.log' os.makedirs(os.path.dirname(logger_filename), exist_ok=True) fh = logging.FileHandler(logger_filename, 'w', 'utf-8') - formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') + formatter = logging.Formatter('%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s') fh.setFormatter(formatter) logger.addHandler(fh) except KeyError: logger.setLevel(logging.ERROR) + +init_logger() class ConsoleReaderBase: """Consol class (aka. client-side python class) for the child. @@ -121,7 +123,6 @@ class ConsoleReaderBase: self.local_echo = True self.pid = os.getpid() - init_logger() logger.info("ConsoleReader started") if cp: @@ -141,43 +142,11 @@ class ConsoleReaderBase: self.__childProcess, _, childPid, self.__tid = win32process.CreateProcess(None, path, None, None, False, 0, None, None, si) - except Exception as e: - logger.info(e) + except: + logger.info(traceback.format_exc()) return - paused = False - - while True: - consinfo = self.consout.GetConsoleScreenBufferInfo() - cursorPos = consinfo['CursorPosition'] - self.send_to_host(self.readConsoleToCursor()) - s = self.get_from_host() - self.write(s) - - if win32process.GetExitCodeProcess(self.__childProcess) != win32con.STILL_ACTIVE: - time.sleep(.1) - try: - win32process.TerminateProcess(self.__childProcess, 0) - except pywintypes.error as e: - """ 'Access denied' happens always? Perhaps if not running as admin (or UAC - enabled under Vista/7). Don't log. Child process will exit regardless when - calling sys.exit - """ - if e.args[0] != winerror.ERROR_ACCESS_DENIED: - logger.info(e) - return - - if cursorPos.Y > maxconsoleY and not paused: - logger.info('cursorPos %s' % cursorPos) - self.suspendThread() - paused = True - - if cursorPos.Y <= maxconsoleY and paused: - logger.info('cursorPos %s' % cursorPos) - self.resumeThread() - paused = False - - time.sleep(.1) + self.read_loop() except: logger.error(traceback.format_exc()) time.sleep(.1) @@ -186,6 +155,44 @@ class ConsoleReaderBase: self.send_to_host(self.readConsoleToCursor()) time.sleep(.1) self.close_connection() + logger.info('Console finished.') + + def read_loop(self): + paused = False + + while True: + consinfo = self.consout.GetConsoleScreenBufferInfo() + cursorPos = consinfo['CursorPosition'] + self.send_to_host(self.readConsoleToCursor()) + s = self.get_from_host() + self.write(s) + + if win32process.GetExitCodeProcess(self.__childProcess) != win32con.STILL_ACTIVE: + logger.info('Child finished.') + time.sleep(.1) + try: + win32process.TerminateProcess(self.__childProcess, 0) + except pywintypes.error as e: + """ 'Access denied' happens always? Perhaps if not running as admin (or UAC + enabled under Vista/7). Don't log. Child process will exit regardless when + calling sys.exit + """ + if e.args[0] != winerror.ERROR_ACCESS_DENIED: + logger.info(e) + return + + if cursorPos.Y > maxconsoleY and not paused: + logger.info('cursorPos %s' % cursorPos) + self.suspendThread() + paused = True + + if cursorPos.Y <= maxconsoleY and paused: + logger.info('cursorPos %s' % cursorPos) + self.resumeThread() + paused = False + + time.sleep(.1) + def write(self, s): """Writes input into the child consoles input buffer.""" @@ -416,6 +423,7 @@ class ConsoleReaderSocket(ConsoleReaderBase): logger.info(f'Socket started at port: {self.port}') # Listen for incoming connections + self.sock.settimeout(5) self.sock.listen(1) self.connection, client_address = self.sock.accept() self.connection.settimeout(.2) @@ -469,7 +477,7 @@ class ConsoleReaderPipe(ConsoleReaderBase): def close_connection(self): if self.pipe: - raise Exception(f'Unimplemented close') + win32file.CloseHandle(self.pipe) def send_to_host(self, msg): # convert to bytes @@ -477,8 +485,12 @@ class ConsoleReaderPipe(ConsoleReaderBase): win32file.WriteFile(self.pipe, msg_bytes) def get_from_host(self): - resp = win32file.ReadFile(self.pipe, 64*1024) - ret = resp[1] - return ret + _, _, avail = win32pipe.PeekNamedPipe(self.pipe, 4096) + if avail > 0: + resp = win32file.ReadFile(self.pipe, 4096) + ret = resp[1] + return ret + else: + return '' diff --git a/wexpect/spawn.py b/wexpect/spawn.py index d427aa3..556a2de 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -71,6 +71,8 @@ import shutil import re import traceback import types +import psutil +import signal import pywintypes import win32process @@ -188,7 +190,7 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None break child_result = ''.join(child_result_list) if withexitstatus: - child.close() + child.wait() return (child_result, child.exitstatus) else: return child_result @@ -196,7 +198,7 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None class SpawnBase: def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, - logfile=None, cwd=None, env=None, codepage=None, echo=True): + logfile=None, cwd=None, env=None, codepage=None, echo=True, **kwargs): """This starts the given command in a child process. This does all the fork/exec type of stuff for a pty. This is called by __init__. If args is empty then command will be parsed (split on spaces) and args will be @@ -211,6 +213,11 @@ class SpawnBase: That may not necessarily be bad because you may haved spawned a child that performs some task; creates no stdout output; and then dies. """ + self.console_process = None + self.console_pid = None + self.child_process = None + self.child_pid = None + self.searcher = None self.ignorecase = False self.before = None @@ -271,6 +278,7 @@ class SpawnBase: self.closed = False self.child_fd = self.startChild(self.args, self.env) + self.get_child_process() self.connect_to_child() def __del__(self): @@ -281,7 +289,7 @@ class SpawnBase: self.terminate() self.disconnect_from_child() except: - pass + traceback.print_exc() def __str__(self): @@ -311,32 +319,64 @@ class SpawnBase: s.append('delaybeforesend: ' + str(self.delaybeforesend)) s.append('delayafterterminate: ' + str(self.delayafterterminate)) return '\n'.join(s) - - def fileno (self): # File-like object. - """There is no child fd.""" + + def get_console_process(self, force=False): + if force or self.console_process is None: + self.console_process = psutil.Process(self.console_pid) + return self.console_process + + def get_child_process(self, force=False): + '''Fetches and returns the child process (and pid) - return 0 + The console starts the *real* child. This function fetches this *real* child's process ID + and process handle. If the console process is slower,(the OS does not grant enough CPU for + that), the child, cannot be started, when we reach this function, therefore the + `self.get_console_process().children()` line will return an empty list. So we ask console's child + in a loop, while, we found a (the) child. + This loop cannot be an infinite loop. If the console's process has error before/during + starting the child. `self.get_console_process().children()` will throw error. + ''' + if force or self.console_process is None: + while True: + children = self.get_console_process().children() + try: + self.child_process = children[0] + except IndexError: + time.sleep(.1) + continue + self.child_pid = self.child_process.pid + return self.child_process def terminate(self, force=False): """Terminate the child. Force not used. """ if not self.isalive(): return True - - win32api.TerminateProcess(self.conproc, 1) + + self.kill() time.sleep(self.delayafterterminate) if not self.isalive(): return True return False - - def close(self, force=True): # File-like object. - """ Closes the child console.""" + + def isalive(self, console=True): + """True if the child is still alive, false otherwise""" - self.closed = self.terminate(force) - if not self.closed: - raise ExceptionPexpect ('close() could not terminate the child using terminate()') - self.closed = True + try: + self.exitstatus = self.child_process.wait(timeout=0) + except psutil.TimeoutExpired: + return True + + def kill(self, sig=signal.SIGTERM): + """Sig == sigint for ctrl-c otherwise the child is terminated.""" + self.child_process.send_signal(sig) + + def wait(self, child=True, console=True): + if child: + self.child_process.wait() + if console: + self.console_process.wait() def read (self, size = -1): # File-like object. """This reads at most "size" bytes from the file (less if the read hits @@ -428,21 +468,6 @@ class SpawnBase: """The child is always created with a console.""" return True - - def kill(self, sig): - """Sig == sigint for ctrl-c otherwise the child is terminated.""" - os.kill(self.conpid, sig) - -# win32api.TerminateProcess(self.conproc, 1) - - def isalive(self, console=True): - """True if the child is still alive, false otherwise""" - - if console: - return win32process.GetExitCodeProcess(self.conproc) == win32con.STILL_ACTIVE - else: - return win32process.GetExitCodeProcess(self.__childProcess) == win32con.STILL_ACTIVE - def write(self, s): # File-like object. @@ -737,17 +762,14 @@ class SpawnBase: class SpawnPipe(SpawnBase): - - - def pipe_client(self, conpid): - pipe_name = 'wexpect_{}'.format(conpid) - pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) - print('Trying to connect to pipe: {}'.format(pipe_full_path)) - quit = False - while not quit: + def connect_to_child(self): + pipe_name = 'wexpect_{}'.format(self.console_pid) + pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) +# print('Trying to connect to pipe: {}'.format(pipe_full_path)) + while True: try: - handle = win32file.CreateFile( + self.pipe = win32file.CreateFile( pipe_full_path, win32file.GENERIC_READ | win32file.GENERIC_WRITE, 0, @@ -756,30 +778,66 @@ class SpawnPipe(SpawnBase): 0, None ) - print("pipe found!") - res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) - if res == 0: - print(f"SetNamedPipeHandleState return code: {res}") - while True: - resp = win32file.ReadFile(handle, 64*1024) - print(f"message: {resp}") - win32file.WriteFile(handle, b'back') +# print("pipe found!") + res = win32pipe.SetNamedPipeHandleState(self.pipe, win32pipe.PIPE_READMODE_MESSAGE, None, None) +# if res == 0: +# print(f"SetNamedPipeHandleState return code: {res}") + return except pywintypes.error as e: if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 - print("no pipe, trying again in a bit later") +# print("no pipe, trying again in a bit later") time.sleep(0.2) - elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 - print("broken pipe, bye bye") - quit = True + else: + raise + + def disconnect_from_child(self): + if self.pipe: + win32file.CloseHandle(self.pipe) + + def read_nonblocking (self, size = 1): + """This reads at most size characters from the child application. If + the end of file is read then an EOF exception will be raised. + + This is not effected by the 'size' parameter, so if you call + read_nonblocking(size=100, timeout=30) and only one character is + available right away then one character will be returned immediately. + It will not wait for 30 seconds for another 99 characters to come in. + + This is a wrapper around Wtty.read(). """ + + if self.closed: + raise ValueError ('I/O operation on closed file in read_nonblocking().') + + try: + # The real child and it's console are two different process. The console dies 0.1 sec + # later to be able to read the child's last output (before EOF). So here we check + # isalive() (which checks the real child.) and try a last read on the console. To catch + # the last output. + # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. + if self.flag_child_finished: + raise EOF('self.flag_child_finished') + if not self.isalive(): + self.flag_child_finished = True + + try: + s = win32file.ReadFile(self.pipe, size)[1] + return s.decode() + except pywintypes.error as e: + if e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 + self.flag_eof = True + raise EOF('broken pipe, bye bye') elif e.args[0] == winerror.ERROR_NO_DATA: '''232 (0xE8) The pipe is being closed. ''' - print("The pipe is being closed.") - quit = True + self.flag_eof = True + raise EOF('The pipe is being closed.') else: raise - + except: + raise + return '' + def send(self, s): """This sends a string to the child process. This returns the number of bytes written. If a log file was set then the data is also written to @@ -788,9 +846,68 @@ class SpawnPipe(SpawnBase): s = str.encode(s) if self.delaybeforesend: time.sleep(self.delaybeforesend) - self.sock.sendall(s) + try: + while True: + win32file.WriteFile(self.pipe, b'back') + except pywintypes.error as e: + if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 + print("no pipe, trying again in a bit later") + time.sleep(0.2) + elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 + print("broken pipe, bye bye") + elif e.args[0] == winerror.ERROR_NO_DATA: + '''232 (0xE8) + The pipe is being closed. + ''' + print("The pipe is being closed.") + else: + raise return len(s) + + def startChild(self, args, env): + si = win32process.GetStartupInfo() + si.dwFlags = win32process.STARTF_USESHOWWINDOW + si.wShowWindow = win32con.SW_HIDE + dirname = os.path.dirname(sys.executable + if getattr(sys, 'frozen', False) else + os.path.abspath(__file__)) + spath = [os.path.dirname(dirname)] + pyargs = ['-c'] + if getattr(sys, 'frozen', False): + # If we are running 'frozen', add library.zip and lib\library.zip + # to sys.path + # py2exe: Needs appropriate 'zipfile' option in setup script and + # 'bundle_files' 3 + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + if os.path.isdir(os.path.join(dirname, 'lib')): + dirname = os.path.join(dirname, 'lib') + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + pyargs.insert(0, '-S') # skip 'import site' + + + pid = win32process.GetCurrentProcessId() + + commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') + if getattr(sys, 'frozen', False) else + os.path.join(os.path.dirname(sys.executable), 'python.exe'), + ' '.join(pyargs), + "import sys;" + f"sys.path = {spath} + sys.path;" + "import wexpect;" + "import time;" + "wexpect.console_reader.logger.info('loggerStart.');" + f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {pid});" + "wexpect.console_reader.logger.info('Console finished2.');" + ) + + _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, + win32process.CREATE_NEW_CONSOLE, None, None, si) + class SpawnSocket(SpawnBase): @@ -820,6 +937,7 @@ class SpawnSocket(SpawnBase): def disconnect_from_child(self): if self.sock: self.sock.close() + self.sock = None def read_nonblocking (self, size = 1): """This reads at most size characters from the child application. If @@ -889,10 +1007,12 @@ class SpawnSocket(SpawnBase): f"sys.path = {spath} + sys.path;" "import wexpect;" "import time;" + "wexpect.console_reader.logger.info('loggerStart.');" f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port});" + "wexpect.console_reader.logger.info('Console finished2.');" ) - self.conproc, _, self.conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, + _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, win32process.CREATE_NEW_CONSOLE, None, None, si) @@ -1079,29 +1199,3 @@ class searcher_string (object): self.start = first_match self.end = self.start + len(self.match) return best_index - - - - -def main(): - try: - p = SpawnSocket('cmd') - - p.sendline(b'ls') - time.sleep(.5) - data = p.expect('>') - print(data) - print(p.before) - data = p.expect('>') - print(data) - print(p.before) - - except: - traceback.print_exc() - finally: - p.terminate() - - -if __name__ == '__main__': - main() - From 54d961489a9ba3bd97cfbfeb274d60361ebc2974 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Wed, 22 Jan 2020 11:17:07 +0100 Subject: [PATCH 07/22] [ADD] logging into spawn class; [FIX] more fixes around process finishing --- wexpect/console_reader.py | 76 ++++++++++++++++++++------------------- wexpect/spawn.py | 66 +++++++++++++++++++++++----------- wexpect/wexpect_util.py | 21 +++++++++++ 3 files changed, 105 insertions(+), 58 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 3ac0398..a971b27 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -42,6 +42,7 @@ import logging import os import traceback import pkg_resources +import psutil from io import StringIO import ctypes @@ -54,6 +55,8 @@ import winerror import win32pipe import socket +from .wexpect_util import init_logger + # # System-wide constants # @@ -72,28 +75,8 @@ except: # pragma: no cover # console manipulation. # logger = logging.getLogger('wexpect') - -def init_logger(): - logger = logging.getLogger('wexpect') - os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' - try: - logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] - try: - logger_filename = os.environ['WEXPECT_LOGGER_FILENAME'] - except KeyError: - pid = os.getpid() - logger_filename = f'./.wlog/wexpect_{pid}' - logger.setLevel(logger_level) - logger_filename = f'{logger_filename}.log' - os.makedirs(os.path.dirname(logger_filename), exist_ok=True) - fh = logging.FileHandler(logger_filename, 'w', 'utf-8') - formatter = logging.Formatter('%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s') - fh.setFormatter(formatter) - logger.addHandler(fh) - except KeyError: - logger.setLevel(logging.ERROR) -init_logger() +init_logger(logger) class ConsoleReaderBase: """Consol class (aka. client-side python class) for the child. @@ -101,7 +84,7 @@ class ConsoleReaderBase: This class initialize the console starts the child in it and reads the console periodically. """ - def __init__(self, path, parent_pid, cp=None, window_size_x=80, window_size_y=25, + def __init__(self, path, host_pid, cp=None, window_size_x=80, window_size_y=25, buffer_size_x=80, buffer_size_y=16000, **kwargs): """Initialize the console starts the child in it and reads the console periodically. @@ -121,7 +104,9 @@ class ConsoleReaderBase: self.consin = None self.consout = None self.local_echo = True - self.pid = os.getpid() + self.console_pid = os.getpid() + self.host_pid = host_pid + self.host_process = psutil.Process(host_pid) logger.info("ConsoleReader started") @@ -151,6 +136,7 @@ class ConsoleReaderBase: logger.error(traceback.format_exc()) time.sleep(.1) finally: + self.terminate_child() time.sleep(.1) self.send_to_host(self.readConsoleToCursor()) time.sleep(.1) @@ -161,26 +147,20 @@ class ConsoleReaderBase: paused = False while True: + if not self.isalive(self.host_process): + logger.info('Host process has been died.') + return + + if win32process.GetExitCodeProcess(self.__childProcess) != win32con.STILL_ACTIVE: + logger.info('Child finished.') + return + consinfo = self.consout.GetConsoleScreenBufferInfo() cursorPos = consinfo['CursorPosition'] self.send_to_host(self.readConsoleToCursor()) s = self.get_from_host() self.write(s) - if win32process.GetExitCodeProcess(self.__childProcess) != win32con.STILL_ACTIVE: - logger.info('Child finished.') - time.sleep(.1) - try: - win32process.TerminateProcess(self.__childProcess, 0) - except pywintypes.error as e: - """ 'Access denied' happens always? Perhaps if not running as admin (or UAC - enabled under Vista/7). Don't log. Child process will exit regardless when - calling sys.exit - """ - if e.args[0] != winerror.ERROR_ACCESS_DENIED: - logger.info(e) - return - if cursorPos.Y > maxconsoleY and not paused: logger.info('cursorPos %s' % cursorPos) self.suspendThread() @@ -192,7 +172,27 @@ class ConsoleReaderBase: paused = False time.sleep(.1) + + def terminate_child(self): + try: + win32process.TerminateProcess(self.__childProcess, 0) + except pywintypes.error as e: + """ 'Access denied' happens always? Perhaps if not running as admin (or UAC + enabled under Vista/7). Don't log. Child process will exit regardless when + calling sys.exit + """ + # if e.args[0] != winerror.ERROR_ACCESS_DENIED: + logger.info(e) + return + + def isalive(self, process): + """True if the child is still alive, false otherwise""" + try: + process.wait(timeout=0) + return False + except psutil.TimeoutExpired: + return True def write(self, s): """Writes input into the child consoles input buffer.""" @@ -439,6 +439,7 @@ class ConsoleReaderSocket(ConsoleReaderBase): def send_to_host(self, msg): # convert to bytes msg_bytes = str.encode(msg) + logger.debug(f'Sending msg: {msg_bytes}') self.connection.sendall(msg_bytes) def get_from_host(self): @@ -482,6 +483,7 @@ class ConsoleReaderPipe(ConsoleReaderBase): def send_to_host(self, msg): # convert to bytes msg_bytes = str.encode(msg) + logger.debug(f'Sending msg: {msg_bytes}') win32file.WriteFile(self.pipe, msg_bytes) def get_from_host(self): diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 556a2de..f18b04b 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -73,23 +73,29 @@ import traceback import types import psutil import signal +import socket +import logging import pywintypes import win32process import win32con -import win32api import win32file import winerror import win32pipe -import socket from .wexpect_util import ExceptionPexpect from .wexpect_util import EOF from .wexpect_util import TIMEOUT from .wexpect_util import split_command_line +from .wexpect_util import init_logger + +logger = logging.getLogger('wexpect') + +init_logger(logger) -def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None, logfile=None, cwd=None, env=None): +def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None, logfile=None, + cwd=None, env=None, **kwargs): """ This function runs the given command; waits for it to finish; then @@ -152,9 +158,9 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None dictionary passed to a callback. """ if timeout == -1: - child = SpawnSocket(command, maxread=2000, logfile=logfile, cwd=cwd, env=env) + child = SpawnSocket(command, maxread=2000, logfile=logfile, cwd=cwd, env=env, **kwargs) else: - child = SpawnSocket(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env) + child = SpawnSocket(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env, **kwargs) if events is not None: patterns = list(events.keys()) responses = list(events.values()) @@ -198,7 +204,7 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None class SpawnBase: def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, - logfile=None, cwd=None, env=None, codepage=None, echo=True, **kwargs): + logfile=None, cwd=None, env=None, codepage=None, echo=True, safe_exit=True, **kwargs): """This starts the given command in a child process. This does all the fork/exec type of stuff for a pty. This is called by __init__. If args is empty then command will be parsed (split on spaces) and args will be @@ -218,6 +224,7 @@ class SpawnBase: self.child_process = None self.child_pid = None + self.safe_exit = safe_exit self.searcher = None self.ignorecase = False self.before = None @@ -260,25 +267,19 @@ class SpawnBase: command_with_path = shutil.which(self.command) if command_with_path is None: - raise ExceptionPexpect ('The command was not found or was not executable: %s.' % self.command) + logger.warning('The command was not found or was not executable: %s.' % self.command) + raise ExceptionPexpect ('The command was not found or was not executable: %s.' % self.command) self.command = command_with_path self.args[0] = self.command self.name = '<' + ' '.join (self.args) + '>' - - if self.cwd is not None: - os.chdir(self.cwd) - - - if self.cwd is not None: - # Restore the original working dir - os.chdir(self.ocwd) self.terminated = False self.closed = False self.child_fd = self.startChild(self.args, self.env) self.get_child_process() + logger.info(f'Child pid: {self.child_pid} Console pid: {self.console_pid}') self.connect_to_child() def __del__(self): @@ -288,6 +289,8 @@ class SpawnBase: try: self.terminate() self.disconnect_from_child() + if self.safe_exit: + self.wait() except: traceback.print_exc() @@ -370,7 +373,10 @@ class SpawnBase: def kill(self, sig=signal.SIGTERM): """Sig == sigint for ctrl-c otherwise the child is terminated.""" - self.child_process.send_signal(sig) + try: + self.child_process.send_signal(sig) + except psutil._exceptions.NoSuchProcess as e: + logger.info('Child has already died. %s', e) def wait(self, child=True, console=True): if child: @@ -766,7 +772,7 @@ class SpawnPipe(SpawnBase): def connect_to_child(self): pipe_name = 'wexpect_{}'.format(self.console_pid) pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) -# print('Trying to connect to pipe: {}'.format(pipe_full_path)) + logger.debug(f'Trying to connect to pipe: {pipe_full_path}') while True: try: self.pipe = win32file.CreateFile( @@ -778,14 +784,14 @@ class SpawnPipe(SpawnBase): 0, None ) -# print("pipe found!") + logger.debug('Pipe found') res = win32pipe.SetNamedPipeHandleState(self.pipe, win32pipe.PIPE_READMODE_MESSAGE, None, None) -# if res == 0: -# print(f"SetNamedPipeHandleState return code: {res}") + if res == 0: + logger.debug(f"SetNamedPipeHandleState return code: {res}") return except pywintypes.error as e: if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 -# print("no pipe, trying again in a bit later") + logger.debug("no pipe, trying again in a bit later") time.sleep(0.2) else: raise @@ -806,6 +812,7 @@ class SpawnPipe(SpawnBase): This is a wrapper around Wtty.read(). """ if self.closed: + logger.info('I/O operation on closed file in read_nonblocking().') raise ValueError ('I/O operation on closed file in read_nonblocking().') try: @@ -815,22 +822,27 @@ class SpawnPipe(SpawnBase): # the last output. # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. if self.flag_child_finished: + logger.info("EOF('self.flag_child_finished')") raise EOF('self.flag_child_finished') if not self.isalive(): self.flag_child_finished = True + logger.info('self.isalive() == False: Child has been died, lets do a last read!') try: s = win32file.ReadFile(self.pipe, size)[1] + logger.debug(f's: {s}') return s.decode() except pywintypes.error as e: if e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 self.flag_eof = True + logger.info("EOF('broken pipe, bye bye')") raise EOF('broken pipe, bye bye') elif e.args[0] == winerror.ERROR_NO_DATA: '''232 (0xE8) The pipe is being closed. ''' self.flag_eof = True + logger.info("EOF('The pipe is being closed.')") raise EOF('The pipe is being closed.') else: raise @@ -905,6 +917,8 @@ class SpawnPipe(SpawnBase): "wexpect.console_reader.logger.info('Console finished2.');" ) + logger.info(f'Console starter command:{commandLine}') + _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, win32process.CREATE_NEW_CONSOLE, None, None, si) @@ -933,6 +947,7 @@ class SpawnSocket(SpawnBase): def connect_to_child(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect((self.host, self.port)) + self.sock.settimeout(.2) def disconnect_from_child(self): if self.sock: @@ -951,6 +966,7 @@ class SpawnSocket(SpawnBase): This is a wrapper around Wtty.read(). """ if self.closed: + logger.info('I/O operation on closed file in read_nonblocking().') raise ValueError ('I/O operation on closed file in read_nonblocking().') try: @@ -960,14 +976,20 @@ class SpawnSocket(SpawnBase): # the last output. # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. if self.flag_child_finished: + logger.info("EOF('self.flag_child_finished')") raise EOF('self.flag_child_finished') if not self.isalive(): self.flag_child_finished = True + logger.info('self.isalive() == False: Child has been died, lets do a last read!') + logger.debug(f'Reading socket...') s = self.sock.recv(size) + logger.debug(f's: {s}') except EOF: self.flag_eof = True raise + except socket.timeout: + return '' return s.decode() @@ -1012,6 +1034,8 @@ class SpawnSocket(SpawnBase): "wexpect.console_reader.logger.info('Console finished2.');" ) + logger.info(f'Console starter command:{commandLine}') + _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, win32process.CREATE_NEW_CONSOLE, None, None, si) diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index ee17e8c..ef532ac 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -40,6 +40,27 @@ Wexpect Copyright (c) 2019 Benedek Racz import re import traceback import sys +import os +import logging + +def init_logger(logger): + os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' + try: + logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] + try: + logger_filename = os.environ['WEXPECT_LOGGER_FILENAME'] + except KeyError: + pid = os.getpid() + logger_filename = f'./.wlog/wexpect_{pid}' + logger.setLevel(logger_level) + logger_filename = f'{logger_filename}.log' + os.makedirs(os.path.dirname(logger_filename), exist_ok=True) + fh = logging.FileHandler(logger_filename, 'w', 'utf-8') + formatter = logging.Formatter('%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) + except KeyError: + logger.setLevel(logging.ERROR) def split_command_line(command_line, escape_char = '^'): """This splits a command line into a list of arguments. It splits arguments From 9998875dd92e62a33db8394c22ce177d8c582d5d Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Wed, 22 Jan 2020 12:23:01 +0100 Subject: [PATCH 08/22] [FIX] add psutil to requirements; [FIX] test_buffer_interface: the assertion should have been delayed --- examples/hello_wexpect.py | 6 ++++-- requirements.txt | 3 ++- tests/test_expect.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/hello_wexpect.py b/examples/hello_wexpect.py index c6ce839..d306bad 100644 --- a/examples/hello_wexpect.py +++ b/examples/hello_wexpect.py @@ -12,7 +12,8 @@ child = wexpect.spawn('cmd.exe') child.expect('>') # Prints the cmd's start message -print(child.before) +print(child.before, end='') +print(child.after, end='') # run list directory command child.sendline('ls') @@ -21,7 +22,8 @@ child.sendline('ls') child.expect('>') # Prints content of the directory -print(child.before) +print(child.before, end='') +print(child.after, end='') # Exit from cmd child.sendline('exit') diff --git a/requirements.txt b/requirements.txt index 5cd9eeb..633ed61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -pywin32>=220 \ No newline at end of file +pywin32>=220 +psutil>=5.0.0 diff --git a/tests/test_expect.py b/tests/test_expect.py index e2dd69a..e721301 100644 --- a/tests/test_expect.py +++ b/tests/test_expect.py @@ -267,7 +267,8 @@ class ExpectTestCase (PexpectTestCase.PexpectTestCase): def test_buffer_interface(self): p = wexpect.spawn('cat', timeout=5) p.sendline ('Hello') - p.expect ('Hello') + p.expect('Hello') + p.expect('lo') assert len(p.buffer) p.buffer = 'Testing' p.sendeof () From 3965fd4f65f04541db4f8addb5440a5c1854465d Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Wed, 22 Jan 2020 12:33:09 +0100 Subject: [PATCH 09/22] [ADD] terminal local echo feature --- wexpect/console_reader.py | 4 ++-- wexpect/spawn.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index a971b27..92363c0 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -85,7 +85,7 @@ class ConsoleReaderBase: """ def __init__(self, path, host_pid, cp=None, window_size_x=80, window_size_y=25, - buffer_size_x=80, buffer_size_y=16000, **kwargs): + buffer_size_x=80, buffer_size_y=16000, local_echo=True, **kwargs): """Initialize the console starts the child in it and reads the console periodically. Args: @@ -103,7 +103,7 @@ class ConsoleReaderBase: self.connection = None self.consin = None self.consout = None - self.local_echo = True + self.local_echo = local_echo self.console_pid = os.getpid() self.host_pid = host_pid self.host_process = psutil.Process(host_pid) diff --git a/wexpect/spawn.py b/wexpect/spawn.py index f18b04b..3c9376c 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -242,6 +242,7 @@ class SpawnBase: self.delimiter = EOF self.cwd = cwd self.env = env + self.echo = echo self.maxread = maxread # max bytes to read at one time into buffer self.delaybeforesend = 0.05 # Sets sleep time used just before sending data to child. Time in seconds. self.delayafterterminate = 0.1 # Sets delay in terminate() method to allow kernel time to update process status. Time in seconds. @@ -913,7 +914,7 @@ class SpawnPipe(SpawnBase): "import wexpect;" "import time;" "wexpect.console_reader.logger.info('loggerStart.');" - f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {pid});" + f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {pid}, local_echo={self.echo});" "wexpect.console_reader.logger.info('Console finished2.');" ) @@ -1030,7 +1031,7 @@ class SpawnSocket(SpawnBase): "import wexpect;" "import time;" "wexpect.console_reader.logger.info('loggerStart.');" - f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port});" + f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port}, local_echo={self.echo});" "wexpect.console_reader.logger.info('Console finished2.');" ) From 9709eb82e905f6d29d220e8c181dccb4cc6d1e49 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Wed, 22 Jan 2020 13:03:53 +0100 Subject: [PATCH 10/22] [ADD] interact feature --- wexpect/console_reader.py | 15 +++++++++++++-- wexpect/spawn.py | 11 ++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 92363c0..8e7c931 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -51,7 +51,7 @@ import win32console import win32process import win32con import win32file -import winerror +import win32gui import win32pipe import socket @@ -85,7 +85,7 @@ class ConsoleReaderBase: """ def __init__(self, path, host_pid, cp=None, window_size_x=80, window_size_y=25, - buffer_size_x=80, buffer_size_y=16000, local_echo=True, **kwargs): + buffer_size_x=80, buffer_size_y=16000, local_echo=True, interact=False, **kwargs): """Initialize the console starts the child in it and reads the console periodically. Args: @@ -131,6 +131,10 @@ class ConsoleReaderBase: logger.info(traceback.format_exc()) return + if interact: + self.interact() + self.interact() + self.read_loop() except: logger.error(traceback.format_exc()) @@ -410,6 +414,13 @@ class ConsoleReaderBase: return s + def interact(self): + """Displays the child console for interaction.""" + + logger.debug('Start interact window') + win32gui.ShowWindow(win32console.GetConsoleWindow(), win32con.SW_SHOW) + + class ConsoleReaderSocket(ConsoleReaderBase): diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 3c9376c..e694550 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -204,7 +204,7 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None class SpawnBase: def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, - logfile=None, cwd=None, env=None, codepage=None, echo=True, safe_exit=True, **kwargs): + logfile=None, cwd=None, env=None, codepage=None, echo=True, safe_exit=True, interact=False, **kwargs): """This starts the given command in a child process. This does all the fork/exec type of stuff for a pty. This is called by __init__. If args is empty then command will be parsed (split on spaces) and args will be @@ -249,6 +249,7 @@ class SpawnBase: self.flag_child_finished = False self.buffer = '' # This is the read buffer. See maxread. self.searchwindowsize = searchwindowsize # Anything before searchwindowsize point is preserved, but not searched. + self.interact = interact # If command is an int type then it may represent a file descriptor. @@ -914,7 +915,7 @@ class SpawnPipe(SpawnBase): "import wexpect;" "import time;" "wexpect.console_reader.logger.info('loggerStart.');" - f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {pid}, local_echo={self.echo});" + f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {pid}, local_echo={self.echo}, interact={self.interact});" "wexpect.console_reader.logger.info('Console finished2.');" ) @@ -927,11 +928,11 @@ class SpawnPipe(SpawnBase): class SpawnSocket(SpawnBase): def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, - logfile=None, cwd=None, env=None, codepage=None, echo=True, port=4321, host='localhost'): + logfile=None, cwd=None, env=None, codepage=None, echo=True, port=4321, host='localhost', interact=False): self.port = port self.host = host super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, - searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo) + searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo, interact=interact) def send(self, s): @@ -1031,7 +1032,7 @@ class SpawnSocket(SpawnBase): "import wexpect;" "import time;" "wexpect.console_reader.logger.info('loggerStart.');" - f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port}, local_echo={self.echo});" + f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port}, local_echo={self.echo}, interact={self.interact});" "wexpect.console_reader.logger.info('Console finished2.');" ) From eda36aa1a1ac09175370e502e850b90b4a5b63ef Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Wed, 22 Jan 2020 13:33:09 +0100 Subject: [PATCH 11/22] [FIX] align sleep times, now all commands arrives after the before one has been processed.[TST][FIX] destructor and constructor testcases --- tests/test_constructor.py | 9 +++++--- tests/test_destructor.py | 46 ++++++++++++++++++--------------------- wexpect/console_reader.py | 4 ++-- wexpect/spawn.py | 4 ++++ wexpect/wexpect_util.py | 1 - 5 files changed, 33 insertions(+), 31 deletions(-) diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 227abff..9ba69fd 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -41,9 +41,12 @@ class TestCaseConstructor(PexpectTestCase.PexpectTestCase): def test_named_parameters (self): '''This tests that named parameters work. ''' - p = wexpect.spawn ('ls',timeout=10) - p = wexpect.spawn (timeout=10, command='ls') - p = wexpect.spawn (args=[], command='ls') + p = wexpect.spawn('ls',timeout=10) + p.wait() + p = wexpect.spawn(timeout=10, command='ls') + p.wait() + p = wexpect.spawn(args=[], command='ls') + p.wait() if __name__ == '__main__': unittest.main() diff --git a/tests/test_destructor.py b/tests/test_destructor.py index 6c62c08..46a4d81 100644 --- a/tests/test_destructor.py +++ b/tests/test_destructor.py @@ -32,35 +32,34 @@ class TestCaseDestructor(PexpectTestCase.PexpectTestCase): return 'SKIP' gc.collect() time.sleep(2) - p1 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p2 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p3 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p4 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - fd_t1 = (p1.child_fd,p2.child_fd,p3.child_fd,p4.child_fd) + p1 = wexpect.spawn('ls', port=4321) + p2 = wexpect.spawn('ls', port=4322) + p3 = wexpect.spawn('ls', port=4323) + p4 = wexpect.spawn('ls', port=4324) p1.expect(wexpect.EOF) p2.expect(wexpect.EOF) p3.expect(wexpect.EOF) p4.expect(wexpect.EOF) - p1.kill(9) - p2.kill(9) - p3.kill(9) - p4.kill(9) + p1.kill() + p2.kill() + p3.kill() + p4.kill() p1 = None p2 = None p3 = None p4 = None gc.collect() time.sleep(2) # Some platforms are slow at gc... Solaris! + - p1 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p2 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p3 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p4 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - fd_t2 = (p1.child_fd,p2.child_fd,p3.child_fd,p4.child_fd) - p1.kill(9) - p2.kill(9) - p3.kill(9) - p4.kill(9) + p1 = wexpect.spawn('ls', port=4321) + p2 = wexpect.spawn('ls', port=4322) + p3 = wexpect.spawn('ls', port=4323) + p4 = wexpect.spawn('ls', port=4324) + p1.kill() + p2.kill() + p3.kill() + p4.kill() del (p1) del (p2) del (p3) @@ -68,13 +67,10 @@ class TestCaseDestructor(PexpectTestCase.PexpectTestCase): gc.collect() time.sleep(2) - p1 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p2 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p3 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - p4 = wexpect.spawn('"%s" hello_world.py' % self.PYTHONBIN) - fd_t3 = (p1.child_fd,p2.child_fd,p3.child_fd,p4.child_fd) - - assert (fd_t1 == fd_t2 == fd_t3), "pty file descriptors not properly garbage collected (fd_t1,fd_t2,fd_t3)=(%s,%s,%s)" % (str(fd_t1),str(fd_t2),str(fd_t3)) + p1 = wexpect.spawn('ls', port=4321) + p2 = wexpect.spawn('ls', port=4322) + p3 = wexpect.spawn('ls', port=4323) + p4 = wexpect.spawn('ls', port=4324) if __name__ == '__main__': diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 8e7c931..3c9ec35 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -175,7 +175,7 @@ class ConsoleReaderBase: self.resumeThread() paused = False - time.sleep(.1) + time.sleep(.02) def terminate_child(self): try: @@ -437,7 +437,7 @@ class ConsoleReaderSocket(ConsoleReaderBase): self.sock.settimeout(5) self.sock.listen(1) self.connection, client_address = self.sock.accept() - self.connection.settimeout(.2) + self.connection.settimeout(.01) logger.info(f'Client connected: {client_address}') except: logger.error(f"Port: {self.port}") diff --git a/wexpect/spawn.py b/wexpect/spawn.py index e694550..a358040 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -990,6 +990,10 @@ class SpawnSocket(SpawnBase): except EOF: self.flag_eof = True raise + except ConnectionResetError: + self.flag_eof = True + logger.info("EOF('ConnectionResetError')") + raise EOF('ConnectionResetError') except socket.timeout: return '' diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index ef532ac..02e13b2 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -44,7 +44,6 @@ import os import logging def init_logger(logger): - os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' try: logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] try: From 149ef50874b37ba7ab5c93679642c108afc4a665 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Wed, 22 Jan 2020 16:59:24 +0100 Subject: [PATCH 12/22] [CLN] logging levels; [FIX] Pipe avail bytes --- wexpect/console_reader.py | 36 +++++++++++++++++++--------- wexpect/spawn.py | 50 ++++++++++++++++++++++++++------------- wexpect/wexpect_util.py | 8 +++++++ 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 3c9ec35..17141dc 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -163,6 +163,10 @@ class ConsoleReaderBase: cursorPos = consinfo['CursorPosition'] self.send_to_host(self.readConsoleToCursor()) s = self.get_from_host() + if s: + logger.debug(f'get_from_host: {s}') + else: + logger.spam(f'get_from_host: {s}') self.write(s) if cursorPos.Y > maxconsoleY and not paused: @@ -349,14 +353,14 @@ class ConsoleReaderBase: consinfo = self.consout.GetConsoleScreenBufferInfo() cursorPos = consinfo['CursorPosition'] - logger.debug('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) + logger.spam('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) isSameX = cursorPos.X == self.__currentReadCo.X isSameY = cursorPos.Y == self.__currentReadCo.Y isSamePos = isSameX and isSameY - logger.debug('isSameY: %r' % isSameY) - logger.debug('isSamePos: %r' % isSamePos) + logger.spam('isSameY: %r' % isSameY) + logger.spam('isSamePos: %r' % isSamePos) if isSameY or not self.lastReadData.endswith('\r\n'): # Read the current slice again @@ -364,7 +368,7 @@ class ConsoleReaderBase: self.__currentReadCo.X = 0 self.__currentReadCo.Y = self.__bufferY - logger.debug('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) + logger.spam('cursor: %r, current: %r' % (cursorPos, self.__currentReadCo)) raw = self.readConsole(self.__currentReadCo, cursorPos) rawlist = [] @@ -379,11 +383,14 @@ class ConsoleReaderBase: self.__bufferY += len(rawlist) - i break - logger.debug('lastReadData: %r' % self.lastReadData) - logger.debug('s: %r' % s) + logger.spam('lastReadData: %r' % self.lastReadData) + if s: + logger.debug('Read: %r' % s) + else: + logger.spam('Read: %r' % s) if isSamePos and self.lastReadData == s: - logger.debug('isSamePos and self.lastReadData == s') + logger.spam('isSamePos and self.lastReadData == s') s = '' if s: @@ -450,7 +457,10 @@ class ConsoleReaderSocket(ConsoleReaderBase): def send_to_host(self, msg): # convert to bytes msg_bytes = str.encode(msg) - logger.debug(f'Sending msg: {msg_bytes}') + if msg_bytes: + logger.debug(f'Sending msg: {msg_bytes}') + else: + logger.spam(f'Sending msg: {msg_bytes}') self.connection.sendall(msg_bytes) def get_from_host(self): @@ -475,7 +485,7 @@ class ConsoleReaderSocket(ConsoleReaderBase): class ConsoleReaderPipe(ConsoleReaderBase): def create_connection(self, **kwargs): - pipe_name = 'wexpect_{}'.format(self.pid) + pipe_name = 'wexpect_{}'.format(self.console_pid) pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) logger.info('Start pipe server: %s', pipe_full_path) self.pipe = win32pipe.CreateNamedPipe( @@ -494,11 +504,15 @@ class ConsoleReaderPipe(ConsoleReaderBase): def send_to_host(self, msg): # convert to bytes msg_bytes = str.encode(msg) - logger.debug(f'Sending msg: {msg_bytes}') + if msg_bytes: + logger.debug(f'Sending msg: {msg_bytes}') + else: + logger.spam(f'Sending msg: {msg_bytes}') win32file.WriteFile(self.pipe, msg_bytes) def get_from_host(self): - _, _, avail = win32pipe.PeekNamedPipe(self.pipe, 4096) + data, avail, bytes_left = win32pipe.PeekNamedPipe(self.pipe, 4096) + logger.spam(f'data: {data} avail:{avail} bytes_left{bytes_left}') if avail > 0: resp = win32file.ReadFile(self.pipe, 4096) ret = resp[1] diff --git a/wexpect/spawn.py b/wexpect/spawn.py index a358040..fd3af2d 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -186,6 +186,7 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None elif callback_result: break else: + logger.warning("TypeError ('The callback must be a string or function type.')") raise TypeError ('The callback must be a string or function type.') event_count = event_count + 1 except TIMEOUT: @@ -244,7 +245,7 @@ class SpawnBase: self.env = env self.echo = echo self.maxread = maxread # max bytes to read at one time into buffer - self.delaybeforesend = 0.05 # Sets sleep time used just before sending data to child. Time in seconds. + self.delaybeforesend = 0.1 # Sets sleep time used just before sending data to child. Time in seconds. self.delayafterterminate = 0.1 # Sets delay in terminate() method to allow kernel time to update process status. Time in seconds. self.flag_child_finished = False self.buffer = '' # This is the read buffer. See maxread. @@ -254,9 +255,11 @@ class SpawnBase: # If command is an int type then it may represent a file descriptor. if type(command) == type(0): - raise ExceptionPexpect ('Command is an int type. If this is a file descriptor then maybe you want to use fdpexpect.fdspawn which takes an existing file descriptor instead of a command string.') + logger.warning("ExceptionPexpect('Command is an int type. If this is a file descriptor then maybe you want to use fdpexpect.fdspawn which takes an existing file descriptor instead of a command string.')") + raise ExceptionPexpect('Command is an int type. If this is a file descriptor then maybe you want to use fdpexpect.fdspawn which takes an existing file descriptor instead of a command string.') if type (args) != type([]): + logger.warning("TypeError ('The argument, args, must be a list.')") raise TypeError ('The argument, args, must be a list.') if args == []: @@ -577,6 +580,7 @@ class SpawnBase: elif type(p) is type(re.compile('')): compiled_pattern_list.append(p) else: + logger.warning("TypeError ('Argument must be one of StringTypes, EOF, TIMEOUT, SRE_Pattern, or a list of those type. %s' % str(type(p)))") raise TypeError ('Argument must be one of StringTypes, EOF, TIMEOUT, SRE_Pattern, or a list of those type. %s' % str(type(p))) return compiled_pattern_list @@ -694,6 +698,7 @@ class SpawnBase: for p in pattern_list: if type(p) not in (str,) and p not in (TIMEOUT, EOF): + logger.warning('Argument must be one of StringTypes, EOF, TIMEOUT, or a list of those type. %s' % str(type(p))) raise TypeError ('Argument must be one of StringTypes, EOF, TIMEOUT, or a list of those type. %s' % str(type(p))) return self.expect_loop(searcher_string(pattern_list), timeout, searchwindowsize) @@ -729,6 +734,7 @@ class SpawnBase: return self.match_index # No match at this point if timeout is not None and end_time < time.time(): + logger.info('Timeout exceeded in expect_any().') raise TIMEOUT ('Timeout exceeded in expect_any().') # Still have time left, so read more data c = self.read_nonblocking(self.maxread) @@ -747,7 +753,8 @@ class SpawnBase: else: self.match = None self.match_index = None - raise EOF (str(e) + '\n' + str(self)) + logger.info(f'EOF: {e}\n{self}') + raise EOF(f'{e}\n{self}') except TIMEOUT as e: self.buffer = incoming self.before = incoming @@ -760,7 +767,8 @@ class SpawnBase: else: self.match = None self.match_index = None - raise TIMEOUT (str(e) + '\n' + str(self)) + logger.info(f'TIMEOUT: {e}\n{self}') + raise TIMEOUT(f'{e}\n{self}') except: self.before = incoming self.after = None @@ -814,7 +822,7 @@ class SpawnPipe(SpawnBase): This is a wrapper around Wtty.read(). """ if self.closed: - logger.info('I/O operation on closed file in read_nonblocking().') + logger.warning('I/O operation on closed file in read_nonblocking().') raise ValueError ('I/O operation on closed file in read_nonblocking().') try: @@ -824,7 +832,7 @@ class SpawnPipe(SpawnBase): # the last output. # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. if self.flag_child_finished: - logger.info("EOF('self.flag_child_finished')") + logger.info('EOF: self.flag_child_finished') raise EOF('self.flag_child_finished') if not self.isalive(): self.flag_child_finished = True @@ -832,7 +840,11 @@ class SpawnPipe(SpawnBase): try: s = win32file.ReadFile(self.pipe, size)[1] - logger.debug(f's: {s}') + + if s: + logger.debug(f'Readed: {s}') + else: + logger.spam(f'Readed: {s}') return s.decode() except pywintypes.error as e: if e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 @@ -861,19 +873,20 @@ class SpawnPipe(SpawnBase): if self.delaybeforesend: time.sleep(self.delaybeforesend) try: - while True: - win32file.WriteFile(self.pipe, b'back') + if s: + logger.debug(f"Writing: {s}") + win32file.WriteFile(self.pipe, s) + logger.spam(f"WriteFile finished.") except pywintypes.error as e: - if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 - print("no pipe, trying again in a bit later") - time.sleep(0.2) - elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 - print("broken pipe, bye bye") + if e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 + logger.info("EOF: broken pipe, bye bye") + raise EOF("broken pipe, bye bye") elif e.args[0] == winerror.ERROR_NO_DATA: '''232 (0xE8) The pipe is being closed. ''' - print("The pipe is being closed.") + logger.info("The pipe is being closed.") + raise EOF("The pipe is being closed.") else: raise return len(s) @@ -984,9 +997,12 @@ class SpawnSocket(SpawnBase): self.flag_child_finished = True logger.info('self.isalive() == False: Child has been died, lets do a last read!') - logger.debug(f'Reading socket...') s = self.sock.recv(size) - logger.debug(f's: {s}') + + if s: + logger.debug(f'Readed: {s}') + else: + logger.spam(f'Readed: {s}') except EOF: self.flag_eof = True raise diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index 02e13b2..2c7e8cf 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -43,6 +43,14 @@ import sys import os import logging +SPAM = 5 +logging.addLevelName(SPAM, "SPAM") +def spam(self, message, *args, **kws): + if self.isEnabledFor(SPAM): + # Yes, logger takes its '*args' as 'args'. + self._log(SPAM, message, args, **kws) +logging.Logger.spam = spam + def init_logger(logger): try: logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] From a0b37ebf287500696d74b02db60eb1b51e215507 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Thu, 23 Jan 2020 08:25:52 +0100 Subject: [PATCH 13/22] [FIX] Pipe console byte decode was missing --- wexpect/console_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 17141dc..4defc1a 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -516,7 +516,7 @@ class ConsoleReaderPipe(ConsoleReaderBase): if avail > 0: resp = win32file.ReadFile(self.pipe, 4096) ret = resp[1] - return ret + return ret.decode() else: return '' From 1ad1ea28b064d8d706ce7402f72638e7388e6660 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Thu, 23 Jan 2020 09:29:49 +0100 Subject: [PATCH 14/22] [FIX] EOF handling, more fixes needed --- wexpect/__init__.py | 2 +- wexpect/console_reader.py | 9 +++++++++ wexpect/spawn.py | 32 ++++++++++++++++++++++++++++---- wexpect/wexpect_util.py | 3 +++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/wexpect/__init__.py b/wexpect/__init__.py index 27ffa56..9f62368 100644 --- a/wexpect/__init__.py +++ b/wexpect/__init__.py @@ -11,7 +11,7 @@ from .console_reader import ConsoleReaderPipe from .spawn import SpawnSocket from .spawn import SpawnPipe -from .spawn import SpawnSocket as spawn +from .spawn import SpawnPipe as spawn from .spawn import run __all__ = ['split_command_line', 'join_args', 'ExceptionPexpect', 'EOF', 'TIMEOUT', diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 4defc1a..f04d0e3 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -56,6 +56,7 @@ import win32pipe import socket from .wexpect_util import init_logger +from .wexpect_util import EOF_CHAR # # System-wide constants @@ -143,6 +144,7 @@ class ConsoleReaderBase: self.terminate_child() time.sleep(.1) self.send_to_host(self.readConsoleToCursor()) + self.sendeof() time.sleep(.1) self.close_connection() logger.info('Console finished.') @@ -427,6 +429,13 @@ class ConsoleReaderBase: logger.debug('Start interact window') win32gui.ShowWindow(win32console.GetConsoleWindow(), win32con.SW_SHOW) + def sendeof(self): + """This sends an EOF to the host. This sends a character which inform the host that child + has been finished, and all of it's output has been send to host. + """ + + self.send_to_host(EOF_CHAR) + class ConsoleReaderSocket(ConsoleReaderBase): diff --git a/wexpect/spawn.py b/wexpect/spawn.py index fd3af2d..db89c81 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -88,6 +88,7 @@ from .wexpect_util import EOF from .wexpect_util import TIMEOUT from .wexpect_util import split_command_line from .wexpect_util import init_logger +from .wexpect_util import EOF_CHAR logger = logging.getLogger('wexpect') @@ -247,7 +248,6 @@ class SpawnBase: self.maxread = maxread # max bytes to read at one time into buffer self.delaybeforesend = 0.1 # Sets sleep time used just before sending data to child. Time in seconds. self.delayafterterminate = 0.1 # Sets delay in terminate() method to allow kernel time to update process status. Time in seconds. - self.flag_child_finished = False self.buffer = '' # This is the read buffer. See maxread. self.searchwindowsize = searchwindowsize # Anything before searchwindowsize point is preserved, but not searched. self.interact = interact @@ -778,6 +778,15 @@ class SpawnBase: class SpawnPipe(SpawnBase): + def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, + logfile=None, cwd=None, env=None, codepage=None, echo=True, port=4321, host='localhost', interact=False): + self.pipe = None + + super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, + searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo, interact=interact) + + + def connect_to_child(self): pipe_name = 'wexpect_{}'.format(self.console_pid) @@ -831,9 +840,9 @@ class SpawnPipe(SpawnBase): # isalive() (which checks the real child.) and try a last read on the console. To catch # the last output. # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. - if self.flag_child_finished: - logger.info('EOF: self.flag_child_finished') - raise EOF('self.flag_child_finished') +# if self.flag_child_finished: +# logger.info('EOF: self.flag_child_finished') +# raise EOF('self.flag_child_finished') if not self.isalive(): self.flag_child_finished = True logger.info('self.isalive() == False: Child has been died, lets do a last read!') @@ -845,6 +854,12 @@ class SpawnPipe(SpawnBase): logger.debug(f'Readed: {s}') else: logger.spam(f'Readed: {s}') + + if b'\x04' in s: + self.flag_eof = True + logger.info("EOF: EOF character has been arrived") + raise EOF('EOF character has been arrived') + return s.decode() except pywintypes.error as e: if e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 @@ -944,6 +959,8 @@ class SpawnSocket(SpawnBase): logfile=None, cwd=None, env=None, codepage=None, echo=True, port=4321, host='localhost', interact=False): self.port = port self.host = host + self.sock = None + super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo, interact=interact) @@ -1003,6 +1020,13 @@ class SpawnSocket(SpawnBase): logger.debug(f'Readed: {s}') else: logger.spam(f'Readed: {s}') + + + if EOF_CHAR in s: + self.flag_eof = True + logger.info("EOF: EOF character has been arrived") + raise EOF('EOF character has been arrived') + except EOF: self.flag_eof = True raise diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index 2c7e8cf..509ba90 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -43,6 +43,9 @@ import sys import os import logging +# platform does not define VEOF so assume CTRL-D +EOF_CHAR = b'\x04' + SPAM = 5 logging.addLevelName(SPAM, "SPAM") def spam(self, message, *args, **kws): From 40d34f4b92b7ac80e90591e195c0d3757a53088d Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Thu, 23 Jan 2020 12:46:59 +0100 Subject: [PATCH 15/22] [CLN] clean startChild function --- wexpect/spawn.py | 176 ++++++++++++++++------------------------------- 1 file changed, 58 insertions(+), 118 deletions(-) diff --git a/wexpect/spawn.py b/wexpect/spawn.py index db89c81..f372b29 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -159,9 +159,9 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None dictionary passed to a callback. """ if timeout == -1: - child = SpawnSocket(command, maxread=2000, logfile=logfile, cwd=cwd, env=env, **kwargs) + child = SpawnPipe(command, maxread=2000, logfile=logfile, cwd=cwd, env=env, **kwargs) else: - child = SpawnSocket(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env, **kwargs) + child = SpawnPipe(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env, **kwargs) if events is not None: patterns = list(events.keys()) responses = list(events.values()) @@ -221,6 +221,7 @@ class SpawnBase: That may not necessarily be bad because you may haved spawned a child that performs some task; creates no stdout output; and then dies. """ + self.host_pid = os.getpid() # That's me self.console_process = None self.console_pid = None self.child_process = None @@ -238,7 +239,6 @@ class SpawnBase: self.status = None # status returned by os.waitpid self.flag_eof = False self.flag_child_finished = False - self.pid = None self.child_fd = -1 # initially closed self.timeout = timeout self.delimiter = EOF @@ -316,7 +316,7 @@ class SpawnBase: s.append('match_index: ' + str(self.match_index)) s.append('exitstatus: ' + str(self.exitstatus)) s.append('flag_eof: ' + str(self.flag_eof)) - s.append('pid: ' + str(self.pid)) + s.append('host_pid: ' + str(self.host_pid)) s.append('child_fd: ' + str(self.child_fd)) s.append('closed: ' + str(self.closed)) s.append('timeout: ' + str(self.timeout)) @@ -327,7 +327,53 @@ class SpawnBase: s.append('delaybeforesend: ' + str(self.delaybeforesend)) s.append('delayafterterminate: ' + str(self.delayafterterminate)) return '\n'.join(s) + + def startChild(self, args, env): + si = win32process.GetStartupInfo() + si.dwFlags = win32process.STARTF_USESHOWWINDOW + si.wShowWindow = win32con.SW_HIDE + dirname = os.path.dirname(sys.executable + if getattr(sys, 'frozen', False) else + os.path.abspath(__file__)) + spath = [os.path.dirname(dirname)] + pyargs = ['-c'] + if getattr(sys, 'frozen', False): + # If we are running 'frozen', add library.zip and lib\library.zip to sys.path + # py2exe: Needs appropriate 'zipfile' option in setup script and 'bundle_files' 3 + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + if os.path.isdir(os.path.join(dirname, 'lib')): + dirname = os.path.join(dirname, 'lib') + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + pyargs.insert(0, '-S') # skip 'import site' + + if getattr(sys, 'frozen', False): + python_executable = os.path.join(dirname, 'python.exe') + else: + python_executable = os.path.join(os.path.dirname(sys.executable), 'python.exe') + + child_class_initializator = self.get_child_class_initializator(args) + + commandLine = '"%s" %s "%s"' % (python_executable, + ' '.join(pyargs), + "import sys;" + f"sys.path = {spath} + sys.path;" + "import wexpect;" + "import time;" + "wexpect.console_reader.logger.info('loggerStart.');" + f"{child_class_initializator}" + "wexpect.console_reader.logger.info('Console finished2.');" + ) + + logger.info(f'Console starter command:{commandLine}') + + _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, + win32process.CREATE_NEW_CONSOLE, None, None, si) + def get_console_process(self, force=False): if force or self.console_process is None: self.console_process = psutil.Process(self.console_pid) @@ -835,17 +881,7 @@ class SpawnPipe(SpawnBase): raise ValueError ('I/O operation on closed file in read_nonblocking().') try: - # The real child and it's console are two different process. The console dies 0.1 sec - # later to be able to read the child's last output (before EOF). So here we check - # isalive() (which checks the real child.) and try a last read on the console. To catch - # the last output. - # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. -# if self.flag_child_finished: -# logger.info('EOF: self.flag_child_finished') -# raise EOF('self.flag_child_finished') - if not self.isalive(): - self.flag_child_finished = True - logger.info('self.isalive() == False: Child has been died, lets do a last read!') + self.isalive() try: s = win32file.ReadFile(self.pipe, size)[1] @@ -906,51 +942,9 @@ class SpawnPipe(SpawnBase): raise return len(s) - def startChild(self, args, env): - si = win32process.GetStartupInfo() - si.dwFlags = win32process.STARTF_USESHOWWINDOW - si.wShowWindow = win32con.SW_HIDE - - dirname = os.path.dirname(sys.executable - if getattr(sys, 'frozen', False) else - os.path.abspath(__file__)) - spath = [os.path.dirname(dirname)] - pyargs = ['-c'] - if getattr(sys, 'frozen', False): - # If we are running 'frozen', add library.zip and lib\library.zip - # to sys.path - # py2exe: Needs appropriate 'zipfile' option in setup script and - # 'bundle_files' 3 - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - if os.path.isdir(os.path.join(dirname, 'lib')): - dirname = os.path.join(dirname, 'lib') - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - pyargs.insert(0, '-S') # skip 'import site' - - - pid = win32process.GetCurrentProcessId() - - commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') - if getattr(sys, 'frozen', False) else - os.path.join(os.path.dirname(sys.executable), 'python.exe'), - ' '.join(pyargs), - "import sys;" - f"sys.path = {spath} + sys.path;" - "import wexpect;" - "import time;" - "wexpect.console_reader.logger.info('loggerStart.');" - f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {pid}, local_echo={self.echo}, interact={self.interact});" - "wexpect.console_reader.logger.info('Console finished2.');" - ) - - logger.info(f'Console starter command:{commandLine}') - - _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, - win32process.CREATE_NEW_CONSOLE, None, None, si) + def get_child_class_initializator(self, args, **kwargs): + child_class_initializator = f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {self.host_pid}, local_echo={self.echo}, interact={self.interact});" + return child_class_initializator class SpawnSocket(SpawnBase): @@ -1002,18 +996,7 @@ class SpawnSocket(SpawnBase): raise ValueError ('I/O operation on closed file in read_nonblocking().') try: - # The real child and it's console are two different process. The console dies 0.1 sec - # later to be able to read the child's last output (before EOF). So here we check - # isalive() (which checks the real child.) and try a last read on the console. To catch - # the last output. - # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. - if self.flag_child_finished: - logger.info("EOF('self.flag_child_finished')") - raise EOF('self.flag_child_finished') - if not self.isalive(): - self.flag_child_finished = True - logger.info('self.isalive() == False: Child has been died, lets do a last read!') - + self.isalive() s = self.sock.recv(size) if s: @@ -1039,52 +1022,9 @@ class SpawnSocket(SpawnBase): return s.decode() - def startChild(self, args, env): - si = win32process.GetStartupInfo() - si.dwFlags = win32process.STARTF_USESHOWWINDOW - si.wShowWindow = win32con.SW_HIDE - - dirname = os.path.dirname(sys.executable - if getattr(sys, 'frozen', False) else - os.path.abspath(__file__)) - spath = [os.path.dirname(dirname)] - pyargs = ['-c'] - if getattr(sys, 'frozen', False): - # If we are running 'frozen', add library.zip and lib\library.zip - # to sys.path - # py2exe: Needs appropriate 'zipfile' option in setup script and - # 'bundle_files' 3 - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - if os.path.isdir(os.path.join(dirname, 'lib')): - dirname = os.path.join(dirname, 'lib') - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - pyargs.insert(0, '-S') # skip 'import site' - - - pid = win32process.GetCurrentProcessId() - - commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') - if getattr(sys, 'frozen', False) else - os.path.join(os.path.dirname(sys.executable), 'python.exe'), - ' '.join(pyargs), - "import sys;" - f"sys.path = {spath} + sys.path;" - "import wexpect;" - "import time;" - "wexpect.console_reader.logger.info('loggerStart.');" - f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port}, local_echo={self.echo}, interact={self.interact});" - "wexpect.console_reader.logger.info('Console finished2.');" - ) - - logger.info(f'Console starter command:{commandLine}') - - _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, - win32process.CREATE_NEW_CONSOLE, None, None, si) - + def get_child_class_initializator(self, args, **kwargs): + child_class_initializator = f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {self.host_pid}, port={self.port}, local_echo={self.echo}, interact={self.interact});" + return child_class_initializator class searcher_re (object): From 0d645c4ddf84574953fa0b2a1b314faececf3bf8 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Thu, 23 Jan 2020 12:53:01 +0100 Subject: [PATCH 16/22] [CLN] Remove non-standard linebreaks --- wexpect/console_reader.py | 6 ------ wexpect/spawn.py | 26 -------------------------- 2 files changed, 32 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index f04d0e3..5dcfba8 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -195,7 +195,6 @@ class ConsoleReaderBase: logger.info(e) return - def isalive(self, process): """True if the child is still alive, false otherwise""" try: @@ -265,7 +264,6 @@ class ConsoleReaderBase: self.__consSize = consinfo['Size'] logger.info('self.__consSize: ' + str(self.__consSize)) self.startCursorPos = consinfo['CursorPosition'] - def parseData(self, s): """Ensures that special characters are interpretted as @@ -344,7 +342,6 @@ class ConsoleReaderBase: return ''.join(buff) - def readConsoleToCursor(self): """Reads from the current read position to the current cursor position and inserts the string into self.__buffer.""" @@ -439,7 +436,6 @@ class ConsoleReaderBase: class ConsoleReaderSocket(ConsoleReaderBase): - def create_connection(self, **kwargs): try: self.port = kwargs['port'] @@ -528,5 +524,3 @@ class ConsoleReaderPipe(ConsoleReaderBase): return ret.decode() else: return '' - - diff --git a/wexpect/spawn.py b/wexpect/spawn.py index f372b29..a6e793d 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -97,7 +97,6 @@ init_logger(logger) def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None, logfile=None, cwd=None, env=None, **kwargs): - """ This function runs the given command; waits for it to finish; then returns all output as a string. STDERR is included in output. If the full @@ -300,7 +299,6 @@ class SpawnBase: traceback.print_exc() def __str__(self): - """This returns a human-readable string that represents the state of the object. """ @@ -462,7 +460,6 @@ class SpawnBase: return self.before def readline (self, size = -1): # File-like object. - """This reads and returns one entire line. A trailing newline is kept in the string, but may be absent when a file ends with an incomplete line. Note: This readline() looks for a \\r\\n pair even on UNIX @@ -481,7 +478,6 @@ class SpawnBase: return self.before def __iter__ (self): # File-like object. - """This is to support iterators over a file-like object. """ @@ -493,7 +489,6 @@ class SpawnBase: raise NotImplementedError def __next__ (self): # File-like object. - """This is to support iterators over a file-like object. """ @@ -509,7 +504,6 @@ class SpawnBase: self.terminate() def readlines (self, sizehint = -1): # File-like object. - """This reads until EOF using readline() and returns a list containing the lines thus read. The optional "sizehint" argument is ignored. """ @@ -534,7 +528,6 @@ class SpawnBase: self.send(s) def writelines (self, sequence): # File-like object. - """This calls write() for each element in the sequence. The sequence can be any iterable object producing strings, typically a list of strings. This does not add line separators There is no return value. @@ -553,7 +546,6 @@ class SpawnBase: return n def sendeof(self): - """This sends an EOF to the child. This sends a character which causes the pending parent output buffer to be sent to the waiting child program without waiting for end-of-line. If it is the first character @@ -583,7 +575,6 @@ class SpawnBase: raise NotImplementedError def compile_pattern_list(self, patterns): - """This compiles a pattern-string or a list of pattern-strings. Patterns must be a StringType, EOF, TIMEOUT, SRE_Pattern, or a list of those. Patterns may also be None which results in an empty list (you @@ -632,7 +623,6 @@ class SpawnBase: return compiled_pattern_list def expect(self, pattern, timeout = -1, searchwindowsize=None): - """This seeks through the stream until a pattern is matched. The pattern is overloaded and may take several types. The pattern can be a StringType, EOF, a compiled re, or a list of any of those types. @@ -712,7 +702,6 @@ class SpawnBase: return self.expect_list(compiled_pattern_list, timeout, searchwindowsize) def expect_list(self, pattern_list, timeout = -1, searchwindowsize = -1): - """This takes a list of compiled regular expressions and returns the index into the pattern_list that matched the child output. The list may also contain EOF or TIMEOUT (which are not compiled regular @@ -726,7 +715,6 @@ class SpawnBase: return self.expect_loop(searcher_re(pattern_list), timeout, searchwindowsize) def expect_exact(self, pattern_list, timeout = -1, searchwindowsize = -1): - """This is similar to expect(), but uses plain string matching instead of compiled regular expressions in 'pattern_list'. The 'pattern_list' may be a string; a list or other sequence of strings; or TIMEOUT and @@ -750,7 +738,6 @@ class SpawnBase: return self.expect_loop(searcher_string(pattern_list), timeout, searchwindowsize) def expect_loop(self, searcher, timeout = -1, searchwindowsize = -1): - """This is the common loop used inside expect. The 'searcher' should be an instance of searcher_re or searcher_string, which describes how and what to search for in the input. @@ -830,9 +817,6 @@ class SpawnPipe(SpawnBase): super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo, interact=interact) - - - def connect_to_child(self): pipe_name = 'wexpect_{}'.format(self.console_pid) @@ -946,7 +930,6 @@ class SpawnPipe(SpawnBase): child_class_initializator = f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {self.host_pid}, local_echo={self.echo}, interact={self.interact});" return child_class_initializator - class SpawnSocket(SpawnBase): def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, @@ -958,7 +941,6 @@ class SpawnSocket(SpawnBase): super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo, interact=interact) - def send(self, s): """This sends a string to the child process. This returns the number of bytes written. If a log file was set then the data is also written to @@ -1028,7 +1010,6 @@ class SpawnSocket(SpawnBase): class searcher_re (object): - """This is regular expression string search helper for the spawn.expect_any() method. @@ -1047,7 +1028,6 @@ class searcher_re (object): """ def __init__(self, patterns): - """This creates an instance that searches for 'patterns' Where 'patterns' may be a list or other sequence of compiled regular expressions, or the EOF or TIMEOUT types.""" @@ -1065,7 +1045,6 @@ class searcher_re (object): self._searches.append((n, s)) def __str__(self): - """This returns a human-readable string that represents the state of the object.""" @@ -1080,7 +1059,6 @@ class searcher_re (object): return '\n'.join(ss) def search(self, buffer, freshlen, searchwindowsize=None): - """This searches 'buffer' for the first occurence of one of the regular expressions. 'freshlen' must indicate the number of bytes at the end of 'buffer' which have not been searched before. @@ -1116,7 +1094,6 @@ class searcher_re (object): class searcher_string (object): - """This is a plain string search helper for the spawn.expect_any() method. Attributes: @@ -1133,7 +1110,6 @@ class searcher_string (object): """ def __init__(self, strings): - """This creates an instance of searcher_string. This argument 'strings' may be a list; a sequence of strings; or the EOF or TIMEOUT types. """ @@ -1150,7 +1126,6 @@ class searcher_string (object): self._strings.append((n, s)) def __str__(self): - """This returns a human-readable string that represents the state of the object.""" @@ -1165,7 +1140,6 @@ class searcher_string (object): return '\n'.join(ss) def search(self, buffer, freshlen, searchwindowsize=None): - """This searches 'buffer' for the first occurence of one of the search strings. 'freshlen' must indicate the number of bytes at the end of 'buffer' which have not been searched before. It helps to avoid From 6a9b086d0bd9eba27779aea315f926bdc61b7905 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Thu, 23 Jan 2020 13:30:46 +0100 Subject: [PATCH 17/22] [CLN] clean non_blocking_read --- wexpect/console_reader.py | 18 ++++++++++- wexpect/spawn.py | 66 +++++++++++++++++---------------------- 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 5dcfba8..8261b50 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -108,6 +108,8 @@ class ConsoleReaderBase: self.console_pid = os.getpid() self.host_pid = host_pid self.host_process = psutil.Process(host_pid) + self.child_process = None + self.child_pid = None logger.info("ConsoleReader started") @@ -125,8 +127,9 @@ class ConsoleReaderBase: try: self.initConsole() si = win32process.GetStartupInfo() - self.__childProcess, _, childPid, self.__tid = win32process.CreateProcess(None, path, None, None, False, + self.__childProcess, _, self.child_pid, self.__tid = win32process.CreateProcess(None, path, None, None, False, 0, None, None, si) + self.child_process = psutil.Process(self.child_pid) except: logger.info(traceback.format_exc()) @@ -141,6 +144,19 @@ class ConsoleReaderBase: logger.error(traceback.format_exc()) time.sleep(.1) finally: + try: + self.terminate_child() + time.sleep(.1) + self.send_to_host(self.readConsoleToCursor()) + self.sendeof() + time.sleep(.1) + self.close_connection() + logger.info('Console finished.') + except: + logger.error(traceback.format_exc()) + time.sleep(.1) + + self.terminate_child() time.sleep(.1) self.send_to_host(self.readConsoleToCursor()) diff --git a/wexpect/spawn.py b/wexpect/spawn.py index a6e793d..1ead324 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -769,7 +769,8 @@ class SpawnBase: if timeout is not None and end_time < time.time(): logger.info('Timeout exceeded in expect_any().') raise TIMEOUT ('Timeout exceeded in expect_any().') - # Still have time left, so read more data + # Still have time left, so read more data + self.isalive() c = self.read_nonblocking(self.maxread) freshlen = len(c) time.sleep (0.01) @@ -863,41 +864,34 @@ class SpawnPipe(SpawnBase): if self.closed: logger.warning('I/O operation on closed file in read_nonblocking().') raise ValueError ('I/O operation on closed file in read_nonblocking().') - + try: - self.isalive() + s = win32file.ReadFile(self.pipe, size)[1] + + if s: + logger.debug(f'Readed: {s}') + else: + logger.spam(f'Readed: {s}') - try: - s = win32file.ReadFile(self.pipe, size)[1] + if b'\x04' in s: + self.flag_eof = True + logger.info("EOF: EOF character has been arrived") + raise EOF('EOF character has been arrived') - if s: - logger.debug(f'Readed: {s}') - else: - logger.spam(f'Readed: {s}') - - if b'\x04' in s: - self.flag_eof = True - logger.info("EOF: EOF character has been arrived") - raise EOF('EOF character has been arrived') - - return s.decode() - except pywintypes.error as e: - if e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 - self.flag_eof = True - logger.info("EOF('broken pipe, bye bye')") - raise EOF('broken pipe, bye bye') - elif e.args[0] == winerror.ERROR_NO_DATA: - '''232 (0xE8) - The pipe is being closed. - ''' - self.flag_eof = True - logger.info("EOF('The pipe is being closed.')") - raise EOF('The pipe is being closed.') - else: - raise - except: - raise - return '' + return s.decode() + except pywintypes.error as e: + if e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 + self.flag_eof = True + logger.info("EOF('broken pipe, bye bye')") + raise EOF('broken pipe, bye bye') + elif e.args[0] == winerror.ERROR_NO_DATA: + '''232 (0xE8): The pipe is being closed. + ''' + self.flag_eof = True + logger.info("EOF('The pipe is being closed.')") + raise EOF('The pipe is being closed.') + else: + raise def send(self, s): """This sends a string to the child process. This returns the number of @@ -978,7 +972,6 @@ class SpawnSocket(SpawnBase): raise ValueError ('I/O operation on closed file in read_nonblocking().') try: - self.isalive() s = self.sock.recv(size) if s: @@ -991,10 +984,7 @@ class SpawnSocket(SpawnBase): self.flag_eof = True logger.info("EOF: EOF character has been arrived") raise EOF('EOF character has been arrived') - - except EOF: - self.flag_eof = True - raise + except ConnectionResetError: self.flag_eof = True logger.info("EOF('ConnectionResetError')") From 46519bb4cf59522478b3e96105ad9fa2dce717a5 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Thu, 23 Jan 2020 13:31:18 +0100 Subject: [PATCH 18/22] [CLN] clean process die --- tests/test_constructor.py | 4 ++-- wexpect/__init__.py | 2 +- wexpect/console_reader.py | 42 +++++++++++++++------------------------ wexpect/spawn.py | 2 ++ 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 9ba69fd..ac1de8e 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -28,9 +28,9 @@ class TestCaseConstructor(PexpectTestCase.PexpectTestCase): the same results for different styles of invoking __init__(). This assumes that the root directory / is static during the test. ''' - p1 = wexpect.spawn('uname -m -n -p -r -s -v') + p1 = wexpect.spawn('uname -m -n -p -r -s -v', timeout=5, port=4321) p1.expect(wexpect.EOF) - p2 = wexpect.spawn('uname', ['-m', '-n', '-p', '-r', '-s', '-v']) + p2 = wexpect.spawn('uname', ['-m', '-n', '-p', '-r', '-s', '-v'], timeout=5, port=4322) p2.expect(wexpect.EOF) self.assertEqual(p1.before, p2.before) self.assertEqual(str(p1).splitlines()[1:9], str(p2).splitlines()[1:9]) diff --git a/wexpect/__init__.py b/wexpect/__init__.py index 9f62368..27ffa56 100644 --- a/wexpect/__init__.py +++ b/wexpect/__init__.py @@ -11,7 +11,7 @@ from .console_reader import ConsoleReaderPipe from .spawn import SpawnSocket from .spawn import SpawnPipe -from .spawn import SpawnPipe as spawn +from .spawn import SpawnSocket as spawn from .spawn import run __all__ = ['split_command_line', 'join_args', 'ExceptionPexpect', 'EOF', 'TIMEOUT', diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index 8261b50..c429133 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -157,14 +157,6 @@ class ConsoleReaderBase: time.sleep(.1) - self.terminate_child() - time.sleep(.1) - self.send_to_host(self.readConsoleToCursor()) - self.sendeof() - time.sleep(.1) - self.close_connection() - logger.info('Console finished.') - def read_loop(self): paused = False @@ -201,14 +193,10 @@ class ConsoleReaderBase: def terminate_child(self): try: - win32process.TerminateProcess(self.__childProcess, 0) - except pywintypes.error as e: - """ 'Access denied' happens always? Perhaps if not running as admin (or UAC - enabled under Vista/7). Don't log. Child process will exit regardless when - calling sys.exit - """ - # if e.args[0] != winerror.ERROR_ACCESS_DENIED: - logger.info(e) + if self.child_process: + self.child_process.kill() + except psutil.NoSuchProcess: + logger.info('The process has already died.') return def isalive(self, process): @@ -477,12 +465,13 @@ class ConsoleReaderSocket(ConsoleReaderBase): def send_to_host(self, msg): # convert to bytes - msg_bytes = str.encode(msg) - if msg_bytes: - logger.debug(f'Sending msg: {msg_bytes}') + if isinstance(msg, str): + msg = str.encode(msg) + if msg: + logger.debug(f'Sending msg: {msg}') else: - logger.spam(f'Sending msg: {msg_bytes}') - self.connection.sendall(msg_bytes) + logger.spam(f'Sending msg: {msg}') + self.connection.sendall(msg) def get_from_host(self): try: @@ -524,12 +513,13 @@ class ConsoleReaderPipe(ConsoleReaderBase): def send_to_host(self, msg): # convert to bytes - msg_bytes = str.encode(msg) - if msg_bytes: - logger.debug(f'Sending msg: {msg_bytes}') + if isinstance(msg, str): + msg = str.encode(msg) + if msg: + logger.debug(f'Sending msg: {msg}') else: - logger.spam(f'Sending msg: {msg_bytes}') - win32file.WriteFile(self.pipe, msg_bytes) + logger.spam(f'Sending msg: {msg}') + win32file.WriteFile(self.pipe, msg) def get_from_host(self): data, avail, bytes_left = win32pipe.PeekNamedPipe(self.pipe, 4096) diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 1ead324..4b29eab 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -291,12 +291,14 @@ class SpawnBase: garbage collects Python objects, not the child console.""" try: + logger.info('Deleting...') self.terminate() self.disconnect_from_child() if self.safe_exit: self.wait() except: traceback.print_exc() + logger.warning(traceback.format_exc()) def __str__(self): """This returns a human-readable string that represents the state of From 51426bc41d3e7f3608599279417245f86fbf8efd Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Fri, 24 Jan 2020 10:11:14 +0100 Subject: [PATCH 19/22] [ADD] WEXPECT_SPAWN_CLASS environment variable set the working spawn class, the default is SpawnSocket if env var not set;[FIX][TST] interact testcases skipped --- tests/test_interact.py | 5 ++++- wexpect/__init__.py | 14 +++++++++++++- wexpect/spawn.py | 23 ++++++++++++++--------- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/tests/test_interact.py b/tests/test_interact.py index 953c733..fc7c16d 100644 --- a/tests/test_interact.py +++ b/tests/test_interact.py @@ -33,6 +33,8 @@ import unittest from . import PexpectTestCase class InteractTestCase(PexpectTestCase.PexpectTestCase): + + @unittest.skipIf(not hasattr(wexpect.spawn, 'interact'), "spawn does not support runtime interact switching.") def test_interact(self): # Path of cmd executable: cmd_exe = 'cmd' @@ -54,7 +56,8 @@ class InteractTestCase(PexpectTestCase.PexpectTestCase): p.stop_interact() self.assertEqual('hello', p.before.splitlines()[1]) - + + @unittest.skipIf(not hasattr(wexpect.spawn, 'interact'), "spawn does not support runtime interact switching.") def test_interact_dead(self): # Path of cmd executable: echo = 'echo hello' diff --git a/wexpect/__init__.py b/wexpect/__init__.py index 27ffa56..1aa7e38 100644 --- a/wexpect/__init__.py +++ b/wexpect/__init__.py @@ -1,5 +1,7 @@ # __init__.py +import os + from .wexpect_util import split_command_line from .wexpect_util import join_args from .wexpect_util import ExceptionPexpect @@ -11,8 +13,18 @@ from .console_reader import ConsoleReaderPipe from .spawn import SpawnSocket from .spawn import SpawnPipe -from .spawn import SpawnSocket as spawn from .spawn import run +try: + spawn_class_name = os.environ['WEXPECT_SPAWN_CLASS'] + try: + spawn = globals()[spawn_class_name] + except KeyError: + print(f'Error: no spawn class: {spawn_class_name}') + print('Using SpawnSocket.') + spawn = SpawnSocket +except KeyError: + spawn = SpawnSocket + __all__ = ['split_command_line', 'join_args', 'ExceptionPexpect', 'EOF', 'TIMEOUT', 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'SpawnSocket', 'SpawnPipe', 'run'] diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 4b29eab..388621e 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -249,7 +249,7 @@ class SpawnBase: self.delayafterterminate = 0.1 # Sets delay in terminate() method to allow kernel time to update process status. Time in seconds. self.buffer = '' # This is the read buffer. See maxread. self.searchwindowsize = searchwindowsize # Anything before searchwindowsize point is preserved, but not searched. - self.interact = interact + self.interact_state = interact # If command is an int type then it may represent a file descriptor. @@ -356,7 +356,15 @@ class SpawnBase: else: python_executable = os.path.join(os.path.dirname(sys.executable), 'python.exe') - child_class_initializator = self.get_child_class_initializator(args) + self.console_class_parameters.update({ + 'host_pid': self.host_pid, + 'local_echo': self.echo, + 'interact': self.interact_state + }) + console_class_parameters_kv_pairs = [f'{k}={v}' for k,v in self.console_class_parameters.items() ] + console_class_parameters_str = ', '.join(console_class_parameters_kv_pairs) + + child_class_initializator = f"wexpect.{self.console_class_name}(wexpect.join_args({args}), {console_class_parameters_str});" commandLine = '"%s" %s "%s"' % (python_executable, ' '.join(pyargs), @@ -817,6 +825,8 @@ class SpawnPipe(SpawnBase): def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, logfile=None, cwd=None, env=None, codepage=None, echo=True, port=4321, host='localhost', interact=False): self.pipe = None + self.console_class_name = 'ConsoleReaderPipe' + self.console_class_parameters = {} super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo, interact=interact) @@ -922,9 +932,6 @@ class SpawnPipe(SpawnBase): raise return len(s) - def get_child_class_initializator(self, args, **kwargs): - child_class_initializator = f"wexpect.ConsoleReaderPipe(wexpect.join_args({args}), {self.host_pid}, local_echo={self.echo}, interact={self.interact});" - return child_class_initializator class SpawnSocket(SpawnBase): @@ -933,6 +940,8 @@ class SpawnSocket(SpawnBase): self.port = port self.host = host self.sock = None + self.console_class_name = 'ConsoleReaderPipe' + self.console_class_parameters = {'port': port} super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo, interact=interact) @@ -996,10 +1005,6 @@ class SpawnSocket(SpawnBase): return s.decode() - def get_child_class_initializator(self, args, **kwargs): - child_class_initializator = f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {self.host_pid}, port={self.port}, local_echo={self.echo}, interact={self.interact});" - return child_class_initializator - class searcher_re (object): """This is regular expression string search helper for the From 80f63d47efd4d848edff8f44e728b8ea96ae9655 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Fri, 24 Jan 2020 10:35:23 +0100 Subject: [PATCH 20/22] [FIX] process termination and exitstatus --- tests/test_isalive.py | 4 +++- wexpect/spawn.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_isalive.py b/tests/test_isalive.py index ca5817f..3e2707e 100644 --- a/tests/test_isalive.py +++ b/tests/test_isalive.py @@ -52,6 +52,7 @@ class IsAliveTestCase(PexpectTestCase.PexpectTestCase): def test_expect_isalive_dead_after_normal_termination (self): p = wexpect.spawn('ls', timeout=15) p.expect(wexpect.EOF) + time.sleep(.5) assert not p.isalive() def test_expect_isalive_dead_after_SIGHUP(self): @@ -77,7 +78,7 @@ class IsAliveTestCase(PexpectTestCase.PexpectTestCase): def test_expect_isalive_dead_after_SIGKILL(self): p = wexpect.spawn('cat', timeout=5) assert p.isalive() - p.kill(9) + p.kill() p.expect(wexpect.EOF) assert not p.isalive() @@ -99,6 +100,7 @@ class IsAliveTestCase(PexpectTestCase.PexpectTestCase): assert p.isalive() p.sendeof() p.expect(wexpect.EOF) + time.sleep(.5) assert not p.isalive() assert not p.isalive() diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 388621e..1376cd0 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -441,7 +441,8 @@ class SpawnBase: if child: self.child_process.wait() if console: - self.console_process.wait() + self.exitstatus = self.console_process.wait() + return self.exitstatus def read (self, size = -1): # File-like object. """This reads at most "size" bytes from the file (less if the read hits From e62dfabc829cd3302da2e0cbc4efdb68a5413ca2 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Fri, 24 Jan 2020 16:24:03 +0100 Subject: [PATCH 21/22] [FIX] process termination: do not terminate a process, which has not been started. --- wexpect/spawn.py | 13 +++++++++---- wexpect/wexpect_util.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/wexpect/spawn.py b/wexpect/spawn.py index 1376cd0..33ed162 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -292,10 +292,11 @@ class SpawnBase: try: logger.info('Deleting...') - self.terminate() - self.disconnect_from_child() - if self.safe_exit: - self.wait() + if self.child_process is not None: + self.terminate() + self.disconnect_from_child() + if self.safe_exit: + self.wait() except: traceback.print_exc() logger.warning(traceback.format_exc()) @@ -425,6 +426,10 @@ class SpawnBase: def isalive(self, console=True): """True if the child is still alive, false otherwise""" + if self.child_process is None: + # Child process has not been started... Not alive + return False + try: self.exitstatus = self.child_process.wait(timeout=0) except psutil.TimeoutExpired: diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index 509ba90..cbedbbc 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -162,7 +162,7 @@ class ExceptionPexpect(Exception): def __filter_not_wexpect(self, trace_list_item): """This returns True if list item 0 the string 'wexpect.py' in it. """ - if trace_list_item[0].find('wexpect.py') == -1: + if trace_list_item[0].find('spawn.py') == -1: return True else: return False From fa38c449478d956e5a26cf7eb549d4cc908db2ff Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Fri, 24 Jan 2020 17:25:12 +0100 Subject: [PATCH 22/22] [REF] spawn renamed to host not to hide spawn class; [FIX][TST] More misc test fixed --- tests/test_misc.py | 21 +++++++++++---------- wexpect/__init__.py | 6 +++--- wexpect/console_reader.py | 1 - wexpect/{spawn.py => host.py} | 9 +++++++-- wexpect/wexpect_util.py | 2 +- 5 files changed, 22 insertions(+), 17 deletions(-) rename wexpect/{spawn.py => host.py} (99%) diff --git a/tests/test_misc.py b/tests/test_misc.py index ea94c12..d9eaf37 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -198,10 +198,11 @@ class TestCaseMisc(PexpectTestCase.PexpectTestCase): def test_isalive(self): " check isalive() before and after EOF. (True, False) " child = wexpect.spawn('cat') - assert child.isalive() is True + self.assertTrue(child.isalive()) child.sendeof() child.expect(wexpect.EOF) - assert child.isalive() is False + time.sleep(.5) + self.assertFalse(child.isalive()) def test_bad_type_in_expect(self): " expect() does not accept dictionary arguments. " @@ -235,14 +236,14 @@ class TestCaseMisc(PexpectTestCase.PexpectTestCase): # given, given_words = ['alpha', 'beta', 'gamma', 'delta', ] given_search = given_words - if searcher == wexpect.searcher_re: + if searcher == wexpect.host.searcher_re: given_search = [re.compile(word) for word in given_words] if plus is not None: given_search = given_search + [plus] search_string = searcher(given_search) basic_fmt = '\n {0}: {1}' fmt = basic_fmt - if searcher is wexpect.searcher_re: + if searcher is wexpect.host.searcher_re: fmt = '\n {0}: re.compile({1})' expected_output = '{0}:'.format(searcher.__name__) idx = 0 @@ -260,27 +261,27 @@ class TestCaseMisc(PexpectTestCase.PexpectTestCase): def test_searcher_as_string(self): " check searcher_string(..).__str__() " - self._test_searcher_as(wexpect.searcher_string) + self._test_searcher_as(wexpect.host.searcher_string) def test_searcher_as_string_with_EOF(self): " check searcher_string(..).__str__() that includes EOF " - self._test_searcher_as(wexpect.searcher_string, plus=wexpect.EOF) + self._test_searcher_as(wexpect.host.searcher_string, plus=wexpect.EOF) def test_searcher_as_string_with_TIMEOUT(self): " check searcher_string(..).__str__() that includes TIMEOUT " - self._test_searcher_as(wexpect.searcher_string, plus=wexpect.TIMEOUT) + self._test_searcher_as(wexpect.host.searcher_string, plus=wexpect.TIMEOUT) def test_searcher_re_as_string(self): " check searcher_re(..).__str__() " - self._test_searcher_as(wexpect.searcher_re) + self._test_searcher_as(wexpect.host.searcher_re) def test_searcher_re_as_string_with_EOF(self): " check searcher_re(..).__str__() that includes EOF " - self._test_searcher_as(wexpect.searcher_re, plus=wexpect.EOF) + self._test_searcher_as(wexpect.host.searcher_re, plus=wexpect.EOF) def test_searcher_re_as_string_with_TIMEOUT(self): " check searcher_re(..).__str__() that includes TIMEOUT " - self._test_searcher_as(wexpect.searcher_re, plus=wexpect.TIMEOUT) + self._test_searcher_as(wexpect.host.searcher_re, plus=wexpect.TIMEOUT) def test_exception_tb(self): " test get_trace() filters away wexpect/__init__.py calls. " diff --git a/wexpect/__init__.py b/wexpect/__init__.py index 1aa7e38..4934c71 100644 --- a/wexpect/__init__.py +++ b/wexpect/__init__.py @@ -11,9 +11,9 @@ from .wexpect_util import TIMEOUT from .console_reader import ConsoleReaderSocket from .console_reader import ConsoleReaderPipe -from .spawn import SpawnSocket -from .spawn import SpawnPipe -from .spawn import run +from .host import SpawnSocket +from .host import SpawnPipe +from .host import run try: spawn_class_name = os.environ['WEXPECT_SPAWN_CLASS'] diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index c429133..7766f86 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -46,7 +46,6 @@ import psutil from io import StringIO import ctypes -import pywintypes import win32console import win32process import win32con diff --git a/wexpect/spawn.py b/wexpect/host.py similarity index 99% rename from wexpect/spawn.py rename to wexpect/host.py index 33ed162..4ba3b5e 100644 --- a/wexpect/spawn.py +++ b/wexpect/host.py @@ -381,7 +381,7 @@ class SpawnBase: logger.info(f'Console starter command:{commandLine}') _, _, self.console_pid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, - win32process.CREATE_NEW_CONSOLE, None, None, si) + win32process.CREATE_NEW_CONSOLE, None, self.cwd, si) def get_console_process(self, force=False): if force or self.console_process is None: @@ -409,6 +409,11 @@ class SpawnBase: continue self.child_pid = self.child_process.pid return self.child_process + + def close(self): # File-like object. + """ Closes the child console.""" + + self.closed = self.terminate() def terminate(self, force=False): """Terminate the child. Force not used. """ @@ -946,7 +951,7 @@ class SpawnSocket(SpawnBase): self.port = port self.host = host self.sock = None - self.console_class_name = 'ConsoleReaderPipe' + self.console_class_name = 'ConsoleReaderSocket' self.console_class_parameters = {'port': port} super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index cbedbbc..7dd038d 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -162,7 +162,7 @@ class ExceptionPexpect(Exception): def __filter_not_wexpect(self, trace_list_item): """This returns True if list item 0 the string 'wexpect.py' in it. """ - if trace_list_item[0].find('spawn.py') == -1: + if trace_list_item[0].find('host.py') == -1: return True else: return False