From e0073ea1adf5c189e2d169c86f7a5a55aa316733 Mon Sep 17 00:00:00 2001 From: Zak V Date: Sat, 3 Oct 2020 13:48:15 -0400 Subject: [PATCH 01/10] Added support for saving git repo info. --- labscript/labscript.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/labscript/labscript.py b/labscript/labscript.py index 39fd574..2a7aa36 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2206,6 +2206,14 @@ def save_labscripts(hdf5_file): info, err = process.communicate() if info or err: hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') + if compiler.save_git_info: + module_filename = os.path.split(path)[1] + git_commands = [['branch', '--show-current'], ['rev-parse', '--verify', 'HEAD'], ['diff', 'HEAD', module_filename]] + for command in git_commands: + process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, startupinfo=startupinfo) + info, err = process.communicate() + hdf5_file[save_path].attrs['git ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') except ImportError: pass except WindowsError if os.name == 'nt' else None: @@ -2540,6 +2548,7 @@ def labscript_cleanup(): compiler.time_markers = {} compiler._PrimaryBLACS = None compiler.save_hg_info = True + compiler.save_git_info = True compiler.shot_properties = {} class compiler(object): @@ -2560,6 +2569,7 @@ class compiler(object): time_markers = {} _PrimaryBLACS = None save_hg_info = True + save_git_info = True shot_properties = {} # safety measure in case cleanup is called before init From df4e17f8ea00b3297a35ec938ba9948566b089b1 Mon Sep 17 00:00:00 2001 From: Zak V Date: Sat, 3 Oct 2020 14:35:59 -0400 Subject: [PATCH 02/10] Added support for save_hg_info and save_git_info options in the labconfig [labscript] section. --- labscript/labscript.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 2a7aa36..737b041 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -31,6 +31,7 @@ import labscript_utils.h5_lock, h5py import labscript_utils.properties +from labscript_utils.labconfig import LabConfig # This imports the default Qt library that other labscript suite code will # import as well, since it all uses qtutils. By having a Qt library already @@ -2547,8 +2548,8 @@ def labscript_cleanup(): compiler.wait_delay = 0 compiler.time_markers = {} compiler._PrimaryBLACS = None - compiler.save_hg_info = True - compiler.save_git_info = True + compiler.save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) + compiler.save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) compiler.shot_properties = {} class compiler(object): @@ -2568,8 +2569,8 @@ class compiler(object): wait_delay = 0 time_markers = {} _PrimaryBLACS = None - save_hg_info = True - save_git_info = True + save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) + save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) shot_properties = {} # safety measure in case cleanup is called before init From 58ed6db264c2ecd3f283bd42df845a83ef5b0c8f Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:28:35 -0400 Subject: [PATCH 03/10] save_labscripts()'s calls to git/hg for a given file now run in parallel. --- labscript/labscript.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/labscript/labscript.py b/labscript/labscript.py index 737b041..1d891e5 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2201,18 +2201,24 @@ def save_labscripts(hdf5_file): hdf5_file.create_dataset(save_path, data=open(path).read()) if compiler.save_hg_info: hg_commands = [['log', '--limit', '1'], ['status'], ['diff']] + process_list = [] for command in hg_commands: process = subprocess.Popen(['hg'] + command + [os.path.split(path)[1]], cwd=os.path.split(path)[0], stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo) + process_list.append(process) + for process, command in zip(process_list, hg_commands): info, err = process.communicate() if info or err: hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] git_commands = [['branch', '--show-current'], ['rev-parse', '--verify', 'HEAD'], ['diff', 'HEAD', module_filename]] + process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo) + process_list.append(process) + for process, command in zip(process_list, git_commands): info, err = process.communicate() hdf5_file[save_path].attrs['git ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') except ImportError: From 9a20221ea008fc7394f07d472f16e08b26fa6d0e Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:30:44 -0400 Subject: [PATCH 04/10] Removed "--verify" from git rev-parse call. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 1d891e5..15b7d13 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2212,7 +2212,7 @@ def save_labscripts(hdf5_file): hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['rev-parse', '--verify', 'HEAD'], ['diff', 'HEAD', module_filename]] + git_commands = [['branch', '--show-current'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, From dc8d917a87e0fe5e6dac351a4cc1eaee3d84514a Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:35:39 -0400 Subject: [PATCH 05/10] Added "git describe --always HEAD" to git commands. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 15b7d13..cacedf7 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2212,7 +2212,7 @@ def save_labscripts(hdf5_file): hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] + git_commands = [['branch', '--show-current'], ['describe', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, From 2025755da06f08503d26719197b0f3cac99cca2f Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:38:10 -0400 Subject: [PATCH 06/10] Updated git/hg error message. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index cacedf7..3eaf3be 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2224,7 +2224,7 @@ def save_labscripts(hdf5_file): except ImportError: pass except WindowsError if os.name == 'nt' else None: - sys.stderr.write('Warning: Cannot save Mercurial data for imported scripts. Check that the hg command can be run from the command line.\n') + sys.stderr.write('Warning: Cannot save version control data for imported scripts. Check that the hg and/or git command can be run from the command line.\n') def write_device_properties(hdf5_file): From 6022d97b9d7b2ec09f341edb21fe6fba4ea1cc66 Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:52:10 -0400 Subject: [PATCH 07/10] Added '--tags' to call to git describe. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 3eaf3be..a8d1752 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2212,7 +2212,7 @@ def save_labscripts(hdf5_file): hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['describe', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] + git_commands = [['branch', '--show-current'], ['describe', '--tags', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, From 0f3f187aeeb3125de6ee959e9c76388d85499695 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 13 Oct 2020 02:01:49 -0400 Subject: [PATCH 08/10] Results from git/hg are now cached for faster shot compilation. --- labscript/labscript.py | 131 ++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 29 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index a8d1752..5337752 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -32,6 +32,7 @@ import labscript_utils.h5_lock, h5py import labscript_utils.properties from labscript_utils.labconfig import LabConfig +from labscript_utils.filewatcher import FileWatcher # This imports the default Qt library that other labscript suite code will # import as well, since it all uses qtutils. By having a Qt library already @@ -70,7 +71,10 @@ startupinfo.dwFlags |= 1 #subprocess.STARTF_USESHOWWINDOW # This variable isn't defined, but apparently it's equal to one. else: startupinfo = None - + +# Extract settings from labconfig +_SAVE_HG_INFO = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) +_SAVE_GIT_INFO = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) class config(object): suppress_mild_warnings = True @@ -2174,8 +2178,88 @@ def generate_connection_table(hdf5_file): else: master_pseudoclock_name = compiler.master_pseudoclock.name dataset.attrs['master_pseudoclock'] = master_pseudoclock_name - - + +# Create a dictionary for caching results from vcs commands. The keys will be +# the paths to files that are saved during save_labscripts(). The values will be +# a list of tuples of the form (command, info, err); see the "Returns" section +# of the _run_vcs_commands() docstring for more info. Also create a FileWatcher +# instance for tracking when vcs results need updating. The callback will +# replace the outdated cache entry with a new list of updated vcs commands and +# outputs. +_vcs_cache = {} +def _file_watcher_callback(name, info, event): + _vcs_cache[name] = _run_vcs_commands(name) + +_file_watcher = FileWatcher(_file_watcher_callback) + +def _run_vcs_commands(path): + """Run some VCS commands on a file and return their output. + + The function is used to gather up version control system information so that + it can be stored in the hdf5 files of shots. This is for convenience and + compliments the full copy of the file already included in the shot file. + + Whether hg and git commands are run is controlled by the `save_hg_info` + and `save_git_info` options in the `[labscript]` section of the labconfig. + + Args: + path (str): The path with file name and extension of the file on which + the commands will be run. The working directory will be set to the + directory containing the specified file. + + Returns: + results (list of (tuple, str, str)): A list of tuples, each + containing information related to one vcs command of the form + (command, info, err). The first entry in that tuple is itself a + tuple of strings which was passed to subprocess.Popen() in order to + run the command. Then info is a string that contains the text + printed to stdout by that command, and err contains the text printed + to stderr by the command. + """ + # Gather together a list of commands to run. + module_directory, module_filename = os.path.split(path) + vcs_commands = [] + if compiler.save_hg_info: + hg_commands = [ + ['log', '--limit', '1'], + ['status'], + ['diff'], + ] + for command in hg_commands: + command = tuple(['hg'] + command + [module_filename]) + vcs_commands.append((command, module_directory)) + if compiler.save_git_info: + git_commands = [ + ['branch', '--show-current'], + ['describe', '--tags', '--always', 'HEAD'], + ['rev-parse', 'HEAD'], + ['diff', 'HEAD', module_filename], + ] + for command in git_commands: + command = tuple(['git'] + command) + vcs_commands.append((command, module_directory)) + + # Now go through and start running the commands. + process_list = [] + for command, module_directory in vcs_commands: + process = subprocess.Popen( + command, + cwd=module_directory, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + startupinfo=startupinfo, + ) + process_list.append((command, process)) + + # Gather up results from the commands issued. + results = [] + for command, process in process_list: + info, err = process.communicate() + info = info.decode('utf-8') + err = err.decode('utf-8') + results.append((command, info, err)) + return results + def save_labscripts(hdf5_file): if compiler.labscript_file is not None: script_text = open(compiler.labscript_file).read() @@ -2199,28 +2283,17 @@ def save_labscripts(hdf5_file): # Doesn't seem to want to double count files if you just import the contents of a file within a module continue hdf5_file.create_dataset(save_path, data=open(path).read()) - if compiler.save_hg_info: - hg_commands = [['log', '--limit', '1'], ['status'], ['diff']] - process_list = [] - for command in hg_commands: - process = subprocess.Popen(['hg'] + command + [os.path.split(path)[1]], cwd=os.path.split(path)[0], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo) - process_list.append(process) - for process, command in zip(process_list, hg_commands): - info, err = process.communicate() - if info or err: - hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') - if compiler.save_git_info: - module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['describe', '--tags', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] - process_list = [] - for command in git_commands: - process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, startupinfo=startupinfo) - process_list.append(process) - for process, command in zip(process_list, git_commands): - info, err = process.communicate() - hdf5_file[save_path].attrs['git ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') + if path not in _file_watcher.files: + # Add file to watch list and create its entry in the cache. + _file_watcher.add_file(path) + _file_watcher_callback(path, None, None) + # Get a reference to the current results list in case + # another thread updates _vcs_cache[path] to point at a new + # list during this loop. + results = _vcs_cache[path] + for command, info, err in results: + attribute_str = command[0] + ' ' + command[1] + hdf5_file[save_path].attrs[attribute_str] = (info + '\n' + err) except ImportError: pass except WindowsError if os.name == 'nt' else None: @@ -2554,8 +2627,8 @@ def labscript_cleanup(): compiler.wait_delay = 0 compiler.time_markers = {} compiler._PrimaryBLACS = None - compiler.save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) - compiler.save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) + compiler.save_hg_info = _SAVE_HG_INFO + compiler.save_git_info = _SAVE_GIT_INFO compiler.shot_properties = {} class compiler(object): @@ -2575,8 +2648,8 @@ class compiler(object): wait_delay = 0 time_markers = {} _PrimaryBLACS = None - save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) - save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) + save_hg_info = _SAVE_HG_INFO + save_git_info = _SAVE_GIT_INFO shot_properties = {} # safety measure in case cleanup is called before init From 903cc4397dae90fb4206daa2edcdf4f37a173a17 Mon Sep 17 00:00:00 2001 From: Zak V Date: Wed, 14 Oct 2020 03:59:16 -0400 Subject: [PATCH 09/10] Worked on some thread safety issues with vcs caching. --- labscript/labscript.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 5337752..fc74f78 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -16,6 +16,7 @@ import sys import subprocess import keyword +import threading from inspect import getcallargs from functools import wraps @@ -2187,8 +2188,10 @@ def generate_connection_table(hdf5_file): # replace the outdated cache entry with a new list of updated vcs commands and # outputs. _vcs_cache = {} +_vcs_cache_rlock = threading.RLock() def _file_watcher_callback(name, info, event): - _vcs_cache[name] = _run_vcs_commands(name) + with _vcs_cache_rlock: + _vcs_cache[name] = _run_vcs_commands(name) _file_watcher = FileWatcher(_file_watcher_callback) @@ -2283,17 +2286,15 @@ def save_labscripts(hdf5_file): # Doesn't seem to want to double count files if you just import the contents of a file within a module continue hdf5_file.create_dataset(save_path, data=open(path).read()) - if path not in _file_watcher.files: - # Add file to watch list and create its entry in the cache. - _file_watcher.add_file(path) - _file_watcher_callback(path, None, None) - # Get a reference to the current results list in case - # another thread updates _vcs_cache[path] to point at a new - # list during this loop. - results = _vcs_cache[path] - for command, info, err in results: - attribute_str = command[0] + ' ' + command[1] - hdf5_file[save_path].attrs[attribute_str] = (info + '\n' + err) + with _vcs_cache_rlock: + if path not in _vcs_cache: + # Add file to watch list and create its entry in the cache. + _file_watcher.add_file(path) + _file_watcher_callback(path, None, None) + # Save the cached vcs output to the file. + for command, info, err in _vcs_cache[path]: + attribute_str = command[0] + ' ' + command[1] + hdf5_file[save_path].attrs[attribute_str] = (info + '\n' + err) except ImportError: pass except WindowsError if os.name == 'nt' else None: From 4716cebf0d5943808f2dd9f8abbf3020ec2cd8bb Mon Sep 17 00:00:00 2001 From: Zak V Date: Thu, 15 Oct 2020 05:47:31 -0400 Subject: [PATCH 10/10] Reduced _vcs_cache_rlock contention in save_labscripts(). --- labscript/labscript.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index fc74f78..887dd66 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2287,10 +2287,12 @@ def save_labscripts(hdf5_file): continue hdf5_file.create_dataset(save_path, data=open(path).read()) with _vcs_cache_rlock: - if path not in _vcs_cache: - # Add file to watch list and create its entry in the cache. - _file_watcher.add_file(path) - _file_watcher_callback(path, None, None) + already_cached = path in _vcs_cache + if not already_cached: + # Add file to watch list and create its entry in the cache. + _file_watcher.add_file(path) + _file_watcher_callback(path, None, None) + with _vcs_cache_rlock: # Save the cached vcs output to the file. for command, info, err in _vcs_cache[path]: attribute_str = command[0] + ' ' + command[1] pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy