Source code for omfit_tree

import warnings

import time

_t0 = time.time()

import sys
import os as _os

# ensure that any `import omfit.xxx` refers to this installation of OMFIT
sys.path.insert(0, _os.path.dirname(_os.path.abspath(__file__)))
from omfit_classes.startup_framework import *

# import utilities
from utils import _available_to_user_math, _available_to_user_util, _available_to_user_plot, _available_to_user_fusion

# ---------------------
# classes under the `class` folder and starting with OMFIT
# ---------------------
_loaded_omfit_class_files = []

import omfit_classes.omfit_base
from omfit_classes.omfit_base import *
from omfit_classes.omfit_base import _moduleAttrs, itemTagsValues
from omfit_classes import unix_os as os
from omfit_classes.omfit_clusters import OMFITclusters

# override sys.exit to prevent external packages from quitting the session
_exit = sys.exit
sys.exit = lambda *args, **kw: None

import omfit_classes.omfit_python
from omfit_classes.omfit_python import *


_k = 0
for _file in sorted(glob.glob(OMFITsrc + '/omfit_classes/omfit_*.py')):
    _python_module = os.path.splitext(os.path.split(_file)[1])[0]
    try:
        exec("import omfit_classes." + _python_module, globals(), locals())
        try:
            getattr(omfit_classes, _python_module).__all__
        except AttributeError:
            raise OMFITexception(
                'Exception in omfit/%s: OMFIT does not allow modules under omfit/classes'
                'not to have an __all__ attribute which explicitly specifies what symbols '
                'will be exported when from <module> import * is used on the module' % _python_module
            )

        # all classes must be available in omfit_base and omfit_python since that's where the OMFITtree is defined
        if _python_module != 'omfit_base':
            for item in getattr(omfit_classes, _python_module).__all__:
                setattr(omfit_classes.omfit_base, item, getattr(getattr(omfit_classes, _python_module), item))

        if _python_module != 'omfit_python':
            for item in getattr(omfit_classes, _python_module).__all__:
                setattr(omfit_classes.omfit_python, item, getattr(getattr(omfit_classes, _python_module), item))

        exec("from omfit_classes import " + _python_module, globals(), locals())
        exec("from omfit_classes." + _python_module + ' import *', globals(), locals())
        _k += 1
        _loaded_omfit_class_files.append(('%2d)%s' % (_k, _python_module.replace('omfit_', ''))).ljust(20))
    except Exception:
        printe('Offending module: ' + _python_module)
        raise
sys.exit = _exit

for item in omfit_classes.omfit_python.__all__:
    setattr(omfit_classes.omfit_base, item, getattr(omfit_python, item))

# backward compatibility with old classes location in `classes` folder instead of `omfit_classes`
for item in list(sys.modules.keys()):
    if item.startswith('omfit_classes.'):
        # allow `import classes.omfit_classname` # how Python environment with local OMFIT installation can use OMFIT classes
        sys.modules[re.sub('^omfit_classes\.', 'classes.', item)] = sys.modules[item]

print('Loaded OMFIT classes:')
while len(_loaded_omfit_class_files):
    print(''.join(_loaded_omfit_class_files[:4]))
    _loaded_omfit_class_files = _loaded_omfit_class_files[4:]


[docs]def reload_python(moduleName, quiet=False): """ This function extends the Python builtin `reload` function to easily reload OMFIT classes >>> reload_python(omfit_classes.omfit_onetwo) :param moduleName: module or module name :param quiet: bool Suppress print statements listing reloaded objects """ from matplotlib.cbook.deprecation import MatplotlibDeprecationWarning warnings.filterwarnings('ignore', category=MatplotlibDeprecationWarning) if not isinstance(moduleName, str): moduleName = moduleName.__name__ if not (moduleName.startswith('omfit_') or moduleName.startswith('omfit_classes.omfit_')): printe('reload_python is meant for class containing modules (in omfit_classes) that start with `omfit_`') printe('Try instead:\n\nreload({0})\nfrom {0} import *'.format(moduleName)) return if moduleName in ['omfit_tree', 'omfit_gui']: printe('Can not reload omfit_tree or omfit_gui') return tmp = {} exec("import %s" % moduleName, tmp) from importlib import reload reload(eval(moduleName)) exec("from " + moduleName + " import *", tmp) # Keep track of reloaded objects to avoid storing them in the persistent OMFITconsoleDict namespace OMFITreloadedDict.update(tmp) # Find objects in the OMFIT tree that should be reloaded OMtr = traverse(OMFIT, string='OMFIT', onlyDict=True, skipDynaLoad=True) OMtr_cls = {} for loc in OMtr: if hasattr(eval(loc), '__class__') and eval(loc).__class__.__name__ in list(tmp.keys()): OMtr_cls.setdefault(eval(loc).__class__.__name__, []).append(loc) # Select only the items that should be reloaded items = list(tmp.keys()) if hasattr(eval(moduleName), '__all__'): items = eval(moduleName).__all__ if '__builtins__' in items: items.remove('__builtins__') # Loop through items to reload updated = {} for item in items: # Loop through python modules for mod in list(sys.modules.keys()): if hasattr(sys.modules[mod], '__dict__') and item in sys.modules[mod].__dict__: sys.modules[mod].__dict__[item] = tmp[item] updated.setdefault(mod, []).append(item) # Loop through OMFIT objects if item in OMtr_cls: for loc in OMtr_cls[item]: if not quiet: print(loc) eval(loc).__class__ = tmp[item] # Update OMFIT data types omfit_classes.omfit_base._updateTypes() if not quiet: printi('*' * 20) printi('Updated python modules') printi('*' * 20) pprinti(updated) printi('') printi('*' * 20) printi('Updated OMFIT tree entries') printi('*' * 20) pprinti(OMtr_cls) return
omfit_classes.omfit_python.reload_python = reload_python # --------------------- # OMFIT main tree # --------------------- _availableModulesCache = {}
[docs]class OMFITmaintree(OMFITproject): _save_method = '_save_with_info' def __init__(self, filename=''): OMFITproject.__init__(self, filename) self._OMFITparent = None self._OMFITkeyName = '' self.prj_options = {}
[docs] def start(self, filename=''): self.clear() self.filename = filename self.reset() self.onlyRunningCopy() OMFIT['MainSettings'].sort()
[docs] def projectName(self): if not len(self.filename): return '' if re.findall('OMFITsave.txt', self.filename): return os.path.split(self.filename.split(os.sep + 'OMFITsave.txt')[0])[1] else: return os.path.splitext(os.path.split(self.filename)[1])[0]
[docs] def onlyRunningCopy(self, deletePID=False): """ :param delete: whether to remove PID from list of running OMFIT processes (this should be done only at exit) :return: return True/False wether this is the only running copy of OMFIT on this computer """ filename = os.sep.join([OMFITsessionsDir, str(os.getpid())]) # OMFITsessionsDir should always exist! -- if not return False if not os.path.exists(OMFITsessionsDir): printe('Something is wrong! OMFITsessionsDir (%s) has been deleted!' % OMFITsessionsDir) return False else: try: # this is in a try/except because # in some extreme circumstances there can be # errors if the system cannot allocate memory if not os.path.exists(filename): open(filename, 'w').close() pids = [] for file in glob.glob(os.sep.join([OMFITsessionsDir, '*'])): pid = os.path.split(file)[1] if is_running(pid): pids.append(pid) elif deletePID: try: os.remove(file) except OSError: pass if deletePID and os.path.exists(filename): os.remove(filename) return len(pids) == 1 except OSError: return False
[docs] def reset(self): # always use saving as .zip as the default self.zip = True # clear self.clear() OMFITconsoleDict.clear() OMFITscriptsDict.clear() # remove all files under current temporary directory try: shutil.rmtree(OMFITcwd) except Exception: pass if not os.path.exists(OMFITcwd): os.makedirs(OMFITcwd) os.chdir(OMFITcwd) # fill special locations super().__setitem__('scratch', OMFITmainscratch()) super().__setitem__('commandBox', OMFITconsoleDict) super().__setitem__('scriptsRun', OMFITscriptsDict) super().__setitem__('shotBookmarks', OMFITshotBookmarks) # create main settings self.addMainSettings(restore='user') OMFIT['MainSettings']['EXPERIMENT']['runid'] = 'sim1' self['MainSettings']['SETUP']['version'] = repo_active_branch_commit self['MainSettings']['SETUP']['python_environment'] = SortedDict(python_environment()) # initialize main scratch self['scratch'].initialize() # set the localhost SERVER.setLocalhost()
[docs] def newProvenanceID(self): self['MainSettings']['EXPERIMENT']['provenanceID'] = omfit_hash(utils_base.now("%Y-%m-%d_%H_%M_%S_%f"))
[docs] def newProjectID(self): self['MainSettings']['EXPERIMENT']['projectID'] = 'projectID__' self['MainSettings']['EXPERIMENT']['projectID'] += repo_active_branch_commit + '__' self['MainSettings']['EXPERIMENT']['projectID'] += utils_base.now("%Y-%m-%d_%H_%M_%S_%f") self['MainSettings']['EXPERIMENT']['projectID'] = self['MainSettings']['EXPERIMENT']['projectID'].replace(' ', '')
[docs] def addMainSettings(self, updateUserSettings=False, restore=''): # read the user namelist file self.userMainSettings = OMFITsettingsDir + os.sep + 'MainSettings.txt' self.userMainSettingsNamelistDump = OMFITsettingsDir + os.sep + 'MainSettingsNamelistDump.txt' self.userMainSettingsNamelist = OMFITsettingsDir + os.sep + 'MainSettingsNamelist.txt' if not os.path.exists(os.path.split(self.userMainSettings)[0]): os.makedirs(os.path.split(self.userMainSettings)[0]) if not os.path.exists(self.userMainSettings): open(self.userMainSettings, 'w').close() updateUserSettings = True if not os.path.exists(self.userMainSettingsNamelist): open(self.userMainSettingsNamelist, 'w').close() updateUserSettings = True # keep project options for k in list(OMFIT.prj_options_choices.keys()): if k not in self.prj_options: if k == 'persistent_projectID': self.prj_options[k] = False else: self.prj_options[k] = '' # force restore of skel main settings if necessary self.apply_bindings() if 'MainSettings' not in self or restore != '': # skeleton settings skelMainSettings = OMFITsrc + os.sep + 'omfit_classes' + os.sep + 'skeleton' + os.sep + 'skeletonMainSettings.txt' self.tmpSkel = OMFITtree(skelMainSettings, quiet=True) if platform.system() == 'Darwin': skelMainSettings = ( OMFITsrc + os.sep + 'omfit_classes' + os.sep + 'skeleton' + os.sep + 'skeletonMainSettingsNamelistOSX.txt' ) else: skelMainSettings = ( OMFITsrc + os.sep + 'omfit_classes' + os.sep + 'skeleton' + os.sep + 'skeletonMainSettingsNamelistUNIX.txt' ) self.tmpSkel['MainSettings'].recursiveUpdate(namelist.NamelistFile(skelMainSettings), overwrite=True) # institution settings if os.path.exists(os.environ.get('OMFIT_INSTITUTION_FILE', '/does not exist')): tmp = namelist.NamelistFile(os.environ['OMFIT_INSTITUTION_FILE']) else: filename = os.sep.join([OMFITsrc, '..', 'institution']) if os.path.exists(filename): tmp = namelist.NamelistFile(filename) else: institution_files = glob.glob(os.sep.join([OMFITsrc, '..', 'institutions', '']) + '*') institution_files.append(None) for filename in institution_files: tmp = namelist.NamelistFile(filename) if 'SETUP' in tmp and 'stats_file' in tmp['SETUP'] and os.path.exists(tmp['SETUP']['stats_file']): break self.tmpSkel['MainSettings'].recursiveUpdate(tmp, overwrite=True) # user settings self.tmpUser = OMFITtree(self.userMainSettings, quiet=True) if 'MainSettings' not in self: self['MainSettings'] = OMFITmainSettings() self['MainSettings'].filename = OMFITcwd + os.sep + os.path.split(self.userMainSettingsNamelist)[1] if not len(self.tmpUser): restore = 'skel' updateUserSettings = True # The main settings are built starting from the skeleton, the user local preference in ~/.OMFIT/ and the edits that the users have made if restore == '': self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) def f_traverse(me): for kid in list(me.keys()): if isinstance(me[kid], dict): f_traverse(me[kid]) if re.match(hide_ptrn, kid): del me[kid] f_traverse(self['MainSettings']) self.add_bindings_to_main_settings() elif restore.startswith('diff_'): restore = restore[5:] tmp = OMFITmainSettings() if restore == 'skel': tmp.recursiveUpdate(self.tmpSkel['MainSettings']) elif restore == 'user' and 'MainSettings' in self.tmpUser: tmp.recursiveUpdate(self.tmpSkel['MainSettings']) tmp.recursiveUpdate(self.tmpUser['MainSettings'], overwrite=True) elif restore == 'S3': tmp.recursiveUpdate(self.tmpSkel['MainSettings']) tmp.recursiveUpdate(self.tmpUser['MainSettings'], overwrite=True) tmp1 = OMFITobject_fromS3(tmp['SETUP']['email'] + "_" + 'MainSettings.txt', s3bucket='omfit') tmp2 = OMFITobject_fromS3(tmp['SETUP']['email'] + "_" + 'MainSettingsNamelist.txt', s3bucket='omfit') shutil.copy2(tmp2.filename, os.path.split(tmp1.filename)[0] + os.sep + os.path.split(tmp2.filename)[1]) tmp0 = OMFITtree(tmp1.filename, quiet=True) tmp.recursiveUpdate(tmp0['MainSettings'], overwrite=True) printi('Loaded MainSettings from the cloud') diffTreeGUI(OMFIT['MainSettings'], tmp) else: self['MainSettings'].clear() if restore == 'skel': self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) elif restore == 'user' and 'MainSettings' in self.tmpUser: self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) self['MainSettings'].recursiveUpdate(self.tmpUser['MainSettings'], overwrite=True) elif restore == 'S3': self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) self['MainSettings'].recursiveUpdate(self.tmpUser['MainSettings'], overwrite=True) tmp1 = OMFITobject_fromS3(self['MainSettings']['SETUP']['email'] + "_" + 'MainSettings.txt', s3bucket='omfit') tmp2 = OMFITobject_fromS3(self['MainSettings']['SETUP']['email'] + "_" + 'MainSettingsNamelist.txt', s3bucket='omfit') shutil.copy2(tmp2.filename, os.path.split(tmp1.filename)[0] + os.sep + os.path.split(tmp2.filename)[1]) tmp = OMFITtree(tmp1.filename, quiet=True) self['MainSettings'].recursiveUpdate(tmp['MainSettings'], overwrite=True) printi('Restored MainSettings from the cloud') self.apply_bindings() # generate unique projectID if that's not already there if 'projectID' not in self['MainSettings']['EXPERIMENT']: self.newProjectID() # generate unique provenanceID if that's not already there if 'provenanceID' not in self['MainSettings']['EXPERIMENT']: self.newProvenanceID() # make sure auto-save is not more often than 15 minutes try: self['MainSettings']['SETUP']['autosave_minutes'] = int(self['MainSettings']['SETUP']['autosave_minutes']) if self['MainSettings']['SETUP']['autosave_minutes'] < 15: raise OMFITexception('Auto-save time must be >= than 15 minutes') except Exception as _excp: printe('Error in setting Auto-save time\n' + repr(_excp)) self['MainSettings']['SETUP']['autosave_minutes'] = 15 # if default_tunnel is None, then set it based on what servers the user has access if self['MainSettings']['SERVER']['default_tunnel'] is None: favorite_tunnels = ['cybele', 'portal', 'cmodws', 'shenma', 'itm_gateway', 'iter_login', 'cfetr'] default_tunnel = 'cybele' if os.path.exists(os.environ['HOME'] + '/.ssh/known_hosts'): with open(os.environ['HOME'] + '/.ssh/known_hosts', 'r') as f: lines = [_f for _f in f.read().strip().split('\n') if _f] known_tunnels = [] for line in lines: try: for server in line.split()[0].split(','): try: SERVER[server] if SERVER(server) in favorite_tunnels: known_tunnels.append(SERVER(server)) except Exception: pass except IndexError: pass for tunnel in favorite_tunnels: if tunnel in known_tunnels: default_tunnel = tunnel break self['MainSettings']['SERVER']['default_tunnel'] = default_tunnel if updateUserSettings: # find differences of current MainSettings with respect to skeleton tmpU = OMFITtree() tmpU['MainSettings'] = copy.deepcopy(OMFIT['MainSettings']) ptrn = self.tmpSkel.pretty_diff(tmpU) # do not store projectID, provenanceID, or runid for k in ['projectID', 'provenanceID', 'runid']: if k in list(ptrn['MainSettings']['EXPERIMENT'].keys()): del ptrn['MainSettings']['EXPERIMENT'][k] # keep only differences tmpU = prune_mask(tmpU, ptrn) # remove entries that are deprecated _clearDeprecatedMainSettings(tmpU['MainSettings']) # remove entries that are not in the skeleton for item in list(tmpU['MainSettings'].keys()): if item in list(tmpU['MainSettings'].keys()) and item not in self.tmpSkel['MainSettings']: del tmpU['MainSettings'][item] if item in list(tmpU['MainSettings'].keys()) and item != 'SERVER': for subitem in list(tmpU['MainSettings'][item].keys()): if ( subitem in list(tmpU['MainSettings'][item].keys()) and subitem not in self.tmpSkel['MainSettings'][item] and subitem != 'KeyBindings' ): del tmpU['MainSettings'][item][subitem] elif subitem == 'KeyBindings': for desc, event in list(tmpU['MainSettings'][item][subitem].items()): if ( desc in global_event_bindings.default_desc_event and global_event_bindings.default_desc_event[desc] == event ): del tmpU['MainSettings'][item][subitem][desc] # save only differences (if there are any) if len(tmpU): tmpU._save(filename=self.userMainSettings, only="['MainSettings']", onlyOMFITsave=True) global OMFITexpressionsReturnNone tmpExp = OMFITexpressionsReturnNone OMFITexpressionsReturnNone = True try: tmpU['MainSettings'].saveas(self.userMainSettingsNamelist) finally: OMFITexpressionsReturnNone = tmpExp self.tmpUser = tmpU printi(self.userMainSettings + ' has been updated') if updateUserSettings == 'S3': OMFITascii(OMFITsettingsDir + os.sep + 'MainSettings.txt').deploy( self['MainSettings']['SETUP']['email'] + "_" + 'MainSettings.txt', s3bucket='omfit' ) OMFITascii(OMFITsettingsDir + os.sep + 'MainSettingsNamelist.txt').deploy( self['MainSettings']['SETUP']['email'] + "_" + 'MainSettingsNamelist.txt', s3bucket='omfit' ) printi('Saved user MainSettings in the cloud') # save a version of MainSettings in plain text if updateUserSettings or not os.path.exists(self.userMainSettingsNamelistDump): self['MainSettings'].deploy(self.userMainSettingsNamelistDump)
def __setitem__(self, key, value): # TODO: handle 'MainSettings' the same way if key in ['scratch', 'commandBox', 'scriptsRun', 'shotBookmarks']: raise RuntimeError(f"OMFIT['{key}'] is write-protected") else: super().__setitem__(key, value) def __delitem__(self, key): if key in ['scratch', 'commandBox', 'scriptsRun', 'shotBookmarks', 'MainSettings']: raise RuntimeError(f"OMFIT['{key}'] is delete-protected") else: super().__delitem__(key)
[docs] def add_bindings_to_main_settings(self): """ Take the descriptions and events from global_events_bindings and insert them in self['MainSettings']['SETUP']['KeyBindings'][desc] = <event> """ printd('Calling add_bindings_to_main_settings', level=2, topic='keybindings') if 'KeyBindings' not in self['MainSettings']['SETUP']: self['MainSettings']['SETUP']['KeyBindings'] = namelist.NamelistName() for di, d in enumerate(global_event_bindings.descs): self['MainSettings']['SETUP']['KeyBindings'][d.replace(' ', '_')] = global_event_bindings.events[di]
[docs] def apply_bindings(self): """ Take the descriptions and events from self['MainSettings']['SETUP']['KeyBindings'] and use them to update the global_event_bindings """ if ('MainSettings' not in self) or ('SETUP' not in self['MainSettings']) or ('KeyBindings' not in self['MainSettings']['SETUP']): printd("No key bindings in MainSettings['SETUP']['KeyBindings'] -> unable to update", topic='keybindings') return for d, e in list(self['MainSettings']['SETUP']['KeyBindings'].items()): try: global_event_bindings.set(d.replace('_', ' '), e) except ValueError: del self['MainSettings']['SETUP']['KeyBindings'][d] self.add_bindings_to_main_settings()
[docs] def save(self, quiet=False, skip_save_errors=False): """ Writes the content of the OMFIT tree to the filesystem using the same filename and zip options of the last saveas :param quiet: whether to print save progress to screen :param skip_save_errors: skip errors when saving objects """ self.filename = self._save_with_info(self.filename, zip=self.zip, quiet=quiet, skip_save_errors=skip_save_errors) return self.filename
def _save_with_info(self, filename, zip=False, quiet=False, updateExistingDir=False, skip_save_errors=False): """ Wrapper function around the OMFITtree._save method which handles the prj_options dictionary that is only part of the main OMFIT tree """ # this method should have the same signature of the ._save() method if filename: # by default the first time the projects directory is created, # assign permissions so that it is readeable but not writeable # by other users. Users can always override these default permissions # once the directory is created. projpath = os.path.split(os.path.split(filename)[0])[0] if not os.path.exists(projpath): # create projects directory os.makedirs(projpath) # by default, remove group and others write permission for perm in [stat.S_IWGRP, stat.S_IWOTH]: st_mode = os.lstat(projpath).st_mode os.chmod(projpath, st_mode & ~perm) # by default, add group and others read and browse permission for perm in [stat.S_IRGRP, stat.S_IROTH, stat.S_IXGRP, stat.S_IXOTH]: st_mode = os.lstat(projpath).st_mode os.chmod(projpath, st_mode | perm) # update list of shots/times from mainSettings and modules settings modules = self.moduleDict() for k in ['device', 'shot', 'time']: self.prj_options[k] = [] for k in ['device', 'shot', 'shots', 'time', 'times']: self.prj_options[k.rstrip('s')].extend(tolist(evalExpr(self['MainSettings']['EXPERIMENT'][k]))) try: for module in list(modules.keys()): mod = eval('OMFIT' + module) self.prj_options.setdefault(k.rstrip('s'), []).extend( tolist(evalExpr(mod['SETTINGS'].get('EXPERIMENT', {}).get(k, None))) ) self.prj_options[k.rstrip('s')] = list([_f for _f in self.prj_options[k.rstrip('s')] if _f]) except Exception as _excp: printe('Error accessing `%s` of module `OMFIT%s` : %s' % (k, module, repr(_excp))) for k in ['device', 'shot', 'time']: self.prj_options[k] = np.unique(list([_f for _f in self.prj_options[k.rstrip('s')] if _f])).tolist() # if the GUI is open update the GUI elements info if OMFITaux['GUI']: commands = [] commandNames = [] for k in range(len(OMFITaux['GUI'].command)): tmp = OMFITaux['GUI'].command[k].get(1.0, tk.END).strip() if len(tmp): commands.append(OMFITaux['GUI'].command[k].get(1.0, tk.END)) commandNames.append(OMFITaux['GUI'].commandNames[k]) if not len(commands): commands.append('') commandNames.append('1') OMFIT.prj_options['commands'] = commands OMFIT.prj_options['commandNames'] = commandNames OMFIT.prj_options['namespace'] = OMFITaux['GUI'].commandBoxNamespace OMFIT.prj_options['notes'] = OMFITaux['GUI'].notes.get(1.0, tk.END).strip() OMFIT.prj_options['console'] = OMFITaux['console'].get() OMFIT.prj_options.setdefault('color', '') if OMFIT.prj_options['color'] not in list(OMFIT.prj_options_choices['color'].keys()): OMFIT.prj_options['color'] = '' if OMFIT.prj_options['type'] not in OMFIT.prj_options_choices['type']: OMFIT.prj_options['type'] = '' try: OMFITaux['hardLinks'] = True self['__GUISAVE__'] = self.prj_options cb = OMFITtree() for k, v in enumerate(self['__GUISAVE__']['commands']): if not v.strip(): continue try: cb['%d.py' % (k + 1)] = OMFITascii(filename='%d.py' % (k + 1), fromString=v.rstrip()) except Exception as _excp: printe(_excp) if list(cb.keys()): self['__COMMANDBOX__'] = cb orig_version = self['MainSettings']['SETUP']['version'] orig_environ = self['MainSettings']['SETUP']['python_environment'] self['MainSettings']['SETUP']['version'] = repo_active_branch_commit self['MainSettings']['SETUP']['python_environment'] = SortedDict(python_environment()) return self._save( filename, zip=zip, quiet=quiet, skipStorage=False, updateExistingDir=updateExistingDir, skip_save_errors=skip_save_errors, ) except Exception: self['MainSettings']['SETUP']['version'] = orig_version self['MainSettings']['SETUP']['python_environment'] = orig_environ raise finally: if '__GUISAVE__' in self: del self['__GUISAVE__'] if '__COMMANDBOX__' in self: del self['__COMMANDBOX__'] OMFITaux['hardLinks'] = False else: raise IOError('Last saved directory was not defined')
[docs] def saveas(self, filename, zip=None, quiet=False, skip_save_errors=False): """ Writes the content of the OMFIT tree to the filesystem and permanently changes the name of the project :param filename: project filename to save to :param zip: whether the save should occur as a zip file :param quiet: whether to print save progress to screen :param skip_save_errors: skip errors when saving objects """ oldZip = self.zip oldFilename = self.filename oldPrj_options = self.prj_options if zip is not None: self.zip = zip self.filename = filename try: return self.save(quiet=quiet, skip_save_errors=skip_save_errors) except Exception: self.zip = oldZip self.filename = oldFilename self.prj_options = oldPrj_options raise
[docs] def loadModule( self, filename, location=None, withSubmodules=True, availableModulesList=None, checkLicense=True, developerMode=None, depth=0, quiet=False, startup_lib=None, **kw, ): r""" Load a module in OMFIT :param filename: * full path to the module to be loaded * if just the module name is provided, this will be loaded from the public modules * remote/branch:module format will load a module from a specific git remote and branch * module:remote/branch format will load a module from a specific git remote and branch :param location: string with the location where to place the module in the OMFIT tree :param withSubmodules: load submodules or not :param availableModulesList: list of available modules generated by ``OMFIT.availableModules()`` If this list is not passed, then the availableModulesList is generated internally :param checkLicense: Check license files at load :param developerMode: Load module with developer mode option (ie. scripts loaded as modifyOriginal) if None then default behavior is set by ``OMFIT['MainSettings']['SETUP']['developer_mode_by_default']`` Note: public OMFIT installation cannot be loaded in developer mode :param depth: parameter used internally by for keeping track of the recursion depth :param quiet: load modules silently or not :param startup_lib: Used internally for executing OMFITlib_startup scripts :param \**kw: additional keywords passed to OMFITmodule() class :return: instance of the loaded module """ if os.sep not in filename or ':' in filename: # if there is a semicolumn in the filename, then a specific remote/branch is been requested if ':' in filename: work_repo = repo.clone() remote, branch = [x for x in filename.split(':') if '/' in x][0].split('/') filename = [x for x in filename.split(':') if '/' not in x][0] work_repo.switch_branch(branch, remote) filename = os.sep.join([work_repo.git_dir, 'modules', filename, 'OMFITsave.txt']) else: # if it's just a module name, load it from the first of the OMFITmodule directories filename = os.sep.join([OMFITmodule.directories()[0], filename, 'OMFITsave.txt']) filename = os.path.abspath(filename) # get list of available modules if availableModulesList is None: availableModulesList = self.availableModules(quiet=True, same_path_as=filename) # handle developerMode switch OMFITsrc = os.path.split(os.path.split(os.path.split(filename)[0])[0])[0] + os.sep + 'omfit' if os.path.exists(os.sep.join([OMFITsrc, '..', 'public'])): developerMode = False if developerMode is None: developerMode = OMFIT['MainSettings']['SETUP']['developer_mode_by_default'] # load the module moduleName = OMFITmodule.info(filename).get('ID', 'unknown_module') if location is None: location = "self['" + moduleName + "']" # handle printing kw.pop('quiet', None) quiet_module = True if not quiet: quiet_module = { 'newline': False, 'clean': False, 'quiet': False, 'style': ' [{sfill}{svoid}] {perc:3.2f}% ' + '*' * (depth + 1) + ' ' + moduleName + '{mess}', } # load module tmp = OMFITmodule(filename, developerMode=developerMode, quiet=quiet_module, **kw) # remove the module filename as there is no point of keeping it once it's loaded from the repo # OMFITtree (and derived classes) filenames are used for saving them external to the project tmp.filename = '' # if loading as a submodule, then place at the top loc = parseLocation(location) if isinstance(eval(buildLocation(loc[:-1])), OMFITmodule): eval(buildLocation(loc[:-1])).insert(0, loc[-1], tmp) else: eval(buildLocation(loc[:-1]))[loc[-1]] = tmp # add extra commit info to MODULE if filename in availableModulesList: for k in _moduleAttrs: if k in availableModulesList[filename]: tmp['SETTINGS']['MODULE'][k] = availableModulesList[filename][k] # fill in missing infos (e.g. when starting a new module) if not tmp['SETTINGS']['MODULE']['commit']: tmp['SETTINGS']['MODULE']['commit'] = repo.get_hash('HEAD') if tmp['SETTINGS']['MODULE']['contact'] is None: tmp['SETTINGS']['MODULE']['contact'] = [] if not len(tmp['SETTINGS']['MODULE']['contact']) and OMFIT['MainSettings']['SETUP']['email']: tmp['SETTINGS']['MODULE']['contact'].append(OMFIT['MainSettings']['SETUP']['email']) # Check license if checkLicense and 'license' in eval(location): license = eval(location)['license'] email_dict = {} with open(license.filename, 'r') as f: for k in f.readlines(): k = str(k) if 'OMFIT_license_email_notify' in k: notify = [x.strip() for x in re.sub(r'OMFIT_license_email_notify\:(.*)', r'\1', k).strip().split(',')] email_dict = { 'email_address': notify, 'email_prompt': 'Please describe planned research using ' + moduleName, 'fromm': self['MainSettings']['SETUP']['email'], } break try: OMFITlicenses[moduleName] = License(moduleName, license.filename, email_dict=email_dict, rootGUI=OMFITaux['rootGUI']) if OMFITaux['rootGUI']: OMFITlicenses[moduleName].check() except LicenseException: exec('del ' + location, globals(), locals()) raise except tk.TclError: # If there is no DISPLAY, allow access pass except IndexError: # If gui element fails to load, allow access pass # load the submodules (will set DEPENDENCIES, SETTINGS['EXPERIMENT'], and honor LIB['OMFITlib_startup'] scripts) if withSubmodules: # find the submodules (one level deep) moduleDict = eval(location).moduleDict(level=0) # save the dependencies depsDict = {} for subMod in moduleDict: depsDict[subMod] = eval(location + subMod)['SETTINGS']['DEPENDENCIES'] # load the submodules if startup_lib is None: startup_lib = [] for subMod in moduleDict: found = False for avMod in availableModulesList: if moduleDict[subMod]['ID'] == availableModulesList[avMod]['ID']: OMFIT.loadModule( availableModulesList[avMod]['path'], location + subMod, withSubmodules=True, availableModulesList=availableModulesList, checkLicense=checkLicense, developerMode=developerMode, depth=depth + 1, quiet=quiet, startup_lib=startup_lib, **kw, ) found = True break if not found: raise IOError( 'Module %s is a submodule of %s, and it could not found in %s' % (moduleDict[subMod]['ID'], moduleName, os.path.split(os.path.split(filename)[0])[0]) ) # restore the dependencies for subMod in moduleDict: for avMod in availableModulesList: if moduleDict[subMod]['ID'] == availableModulesList[avMod]['ID']: eval(location + subMod)['SETTINGS']['DEPENDENCIES'].update(depsDict[subMod]) break # freeze the root['SETTINGS']['EXPERIMENT'] entries if this is a top-level import that is not a submodule if depth == 0 and 'SETTINGS' in eval(location) and 'EXPERIMENT' in eval(location)['SETTINGS']: freeze = True for tmpName in reversed(traverseLocation(location)[:-1]): if eval(tmpName).__class__ is OMFITmodule and tmpName != 'OMFIT': freeze = False if freeze: for item in list(OMFITaux['moduleSkeletonCache']['SETTINGS']['EXPERIMENT'].keys()): eval(location)['SETTINGS']['EXPERIMENT'][item] = evalExpr(eval(location)['SETTINGS']['EXPERIMENT'][item]) # execute startup script (the OMFITmodulePath variable is setup to point to the OMFITsave.txt file of the module that is being loaded) if 'LIB' in eval(location) and 'OMFITlib_startup' in eval(location)['LIB']: startup_lib.append((moduleName, location, eval(location)['LIB']['OMFITlib_startup'])) if depth == 0: for k, (mname, loc, st_lib) in enumerate(startup_lib): if not quiet: if k == 0: printi("Running OMFITlib_startup:", end='') printi(f' {mname}', end='') try: with quiet_environment(): st_lib.runNoGUI() except Exception as _excp: printe("Error in %s['LIB']['OMFITlib_startup'] : " % (loc) + repr(_excp)) if not quiet: printi('') # store settings at load time # NOTE: this is done both here and in OMFITmodule.__init__ because this one will get execution of OMFITlib_startup and the other instead is needed when reloading a module eval(location)._save_settings_at_import() # return the module return eval(location)
[docs] def load(self, filename, persistent_projectID=False): """ loads an OMFITproject in OMFIT :param filename: filename of the project to load (if `-1` then the most recent project is loaded) :param persistent_projectID: whether the projectID should remain the same """ try: # open the last project if filename == '-1': projects = OMFIT.recentProjects() if len(projects): filename = list(projects.keys())[0] persistent_projectID = projects[filename].get('persistent_projectID', False) # keep aside user 'SETUP','SERVER' tmp = {} for item in ['SETUP', 'SERVER']: if item in self['MainSettings']: tmp[item] = copy.deepcopy(self['MainSettings'][item]) self.filename = os.path.abspath(filename) oldDir = os.getcwd() TMPnoCopyToCWD = OMFITaux['noCopyToCWD'] OMFITaux['noCopyToCWD'] = False try: self._load(filename) finally: os.chdir(oldDir) OMFITaux['noCopyToCWD'] = TMPnoCopyToCWD if zipfile.is_zipfile(self.filename): self.zip = True projectName = os.path.split(self.filename)[1] else: self.zip = False projectName = os.path.split(os.path.split(self.filename)[0])[1] # for older projects we calculate provenanceID based on institution and projectName # very old projects do not even have institution info if 'provenanceID' not in self['MainSettings']['EXPERIMENT']: self['MainSettings']['EXPERIMENT']['provenanceID'] = omfit_hash( self['MainSettings']['SETUP'].get('institution', 'old_project') + '_' + projectName ) # SETUP is overwritten (but not version nor python_environment) project_version = self['MainSettings']['SETUP'].get('version', '?') project_environ = self['MainSettings']['SETUP'].get('python_environment', {}) self['MainSettings']['SETUP'] = tmp['SETUP'] self['MainSettings']['SETUP']['version'] = project_version self['MainSettings']['SETUP']['python_environment'] = project_environ # SERVER is updated self['MainSettings'].setdefault('SERVER', NamelistName()).update(tmp.get('SERVER', {})) # load project options self.prj_options.clear() tmp = OMFITproject.info(self.filename) for k in OMFIT.prj_options_choices: self.prj_options[k] = tmp.get(k, '') for whereSave in ['__GUISAVE__', '__COMMANDBOX__']: if whereSave in OMFIT: del OMFIT[whereSave] if self.prj_options['color'] not in list(self.prj_options_choices['color'].keys()): self.prj_options['color'] = '' if self.prj_options['type'] not in self.prj_options_choices['type']: self.prj_options['type'] = '' if 'namespace' not in self.prj_options or not self.prj_options['namespace']: self.prj_options['namespace'] = 'OMFIT' # update projectID self.prj_options['persistent_projectID'] = persistent_projectID if not persistent_projectID: self.newProjectID() # keep MainSettings in check self.updateMainSettings() self['MainSettings'].sort() # set the localhost SERVER.setLocalhost() # remote projects are loaded in OMFITtmpDir # reset the filename to force users to decide how/where to save it if OMFITtmpDir in self.filename: self.filename = '' except Exception: self.start() raise
[docs] def updateCWD(self): global OMFITcwd self._save(filename=OMFITcwd + os.sep + 'OMFITsave.txt', skipStorage=False)
[docs] def updateMainSettings(self): self.addMainSettings() self.keyOrder.remove('scratch') self.keyOrder.append('scratch') self.keyOrder.remove('commandBox') self.keyOrder.append('commandBox') self.keyOrder.remove('scriptsRun') self.keyOrder.append('scriptsRun') self.keyOrder.remove('shotBookmarks') self.keyOrder.append('shotBookmarks') self.keyOrder.remove('MainSettings') self.keyOrder.append('MainSettings')
[docs] def saveMainSettingsSkeleton(self): """ This utility function updates the MainSettingsSkeleton for the current OMFIT installation """ self['MainSettings'].sort() tmp = copy.deepcopy(self['MainSettings']) self.addMainSettings(restore='skel') keys = self['MainSettings'].keys() for k in keys: if k.startswith('__comment'): del self['MainSettings'][k] self['MainSettings'].sort() try: self['MainSettings'].filename = self['MainSettings']['SETUP']['workDir'] + '../' + 'skeletonMainSettingsNamelist.txt' self['MainSettings']['SETUP']['version'] = '?' self['MainSettings']['SETUP']['python_environment'] = {} del self['MainSettings']['EXPERIMENT']['projectID'] del self['MainSettings']['EXPERIMENT']['provenanceID'] self['MainSettings']['SETUP']['browser'] = None self['MainSettings']['SETUP']['Extensions'].clear() # Avoid changes made by institution file self['MainSettings']['SETUP']['stats_file'] = OMFITexpression("os.path.abspath(OMFITsettingsDir+'/OMFITstats.txt')") self['MainSettings']['SETUP']['institution'] = 'PERSONAL' if 'institutionProjectsDir' in self['MainSettings']['SETUP']: del self['MainSettings']['SETUP']['institutionProjectsDir'] self['MainSettings']['EXPERIMENT']['device'] = None for k in tmp['SERVER']: if k not in self['MainSettings']['SERVER']: self['MainSettings']['SERVER'][k] = tmp['SERVER'][k] if isinstance(tmp['SERVER'][k], namelist.NamelistName): self['MainSettings']['SERVER'][k].update(tmp['SERVER'][k]) self['MainSettings']['SERVER']['localhost'] = namelist.NamelistName() self['MainSettings']['SERVER']['localhost']['workDir'] = OMFITexpression("OMFITcwd+os.sep+'remote_runs'+os.sep") self['MainSettings']['SERVER']['localhost']['server'] = 'localhost' self['MainSettings']['SERVER']['localhost']['tunnel'] = '' self['MainSettings']['SERVER']['localhost']['idl'] = '' self['MainSettings']['SERVER']['localhost']['matlab'] = '' self['MainSettings']['SERVER']['default_tunnel'] = None finally: self['MainSettings'].sort() self._save( filename=OMFITsrc + os.sep + 'omfit_classes' + os.sep + 'skeleton' + os.sep + 'skeletonMainSettings.txt', only="['MainSettings']", updateExistingDir=True, ) self['MainSettings'] = tmp
[docs] def availableModules(self=None, quiet=None, directories=None, same_path_as=None, force=False): """ Index available OMFIT modules in directories :param quiet: verbosity :param directories: list of directories to index. If `None` this is taken from OMFIT['MainSettings']['SETUP']['modulesDir'] :param same_path_as: sample OMFITsave.txt path to set directory to index :param force: force rebuild of .modulesInfoCache file :return: This method returns a dictionary with the available OMFIT modules. Each element in the dictionary is a dictionary itself with the details of the available modules. """ if directories is None: directories = OMFITmodule.directories() else: directories = tolist(directories) if same_path_as is None: pass elif 'OMFITsave.txt' in same_path_as and os.path.exists(same_path_as): directories = [os.path.split(os.path.split(same_path_as)[0])[0]] else: raise IOError('%s file is not a valid OMFITsave.txt file' % same_path_as) updatePersistentCache = [] moduleDict = {} for directory in directories: # the user inputs the directory where the modules are if not quiet: tmp = 'Modules directory: ' + directory printi('=' * len(tmp)) printi(tmp) printi('=' * len(tmp)) print('', end='') if not force and os.path.exists(directory + os.sep + '.modulesInfoCache'): try: with open(directory + os.sep + '.modulesInfoCache', 'rb') as f: _availableModulesCache[directory] = pickle.load(f) except Exception: _availableModulesCache[directory] = {} printw('-- Failed to load persistent git modules cache --') else: _availableModulesCache[directory] = {} # modules in directory modules_list_in_dir = sorted( [os.path.abspath(x) for x in glob.glob(os.sep.join([directory, '*', 'OMFITsave.txt']))], key=lambda x: x.lower() ) # try handling git repository of modules twice # first as if git root directory parent directory of modules directory # second as if git root directory is modules directory cases_errors = [] cases = [] if os.path.split(directory)[1] == 'modules': cases.append([os.path.split(directory)[0], 'modules']) cases.append([directory, '']) for repo_dir, modules_subpath in cases: try: # try to access as a git repository repo = OMFITgit(repo_dir) commit = repo("log -1 --pretty='%H'") branch = repo('rev-parse --abbrev-ref HEAD') remote = repo.get_branches().get(branch, {'remote': ''})['remote'] changed_modules = [] # if same remote/branch and commit if ( np.all([k in _availableModulesCache[directory] for k in ['branch', 'remote', commit]]) and _availableModulesCache[directory]['remote'] == remote and _availableModulesCache[directory]['branch'] == branch ): changes = list([_f for _f in repo("ls-files -m --others --exclude-standard " + directory).split('\n') if _f]) for mod_path in modules_list_in_dir: if ( mod_path not in _availableModulesCache[directory][commit] or len(_availableModulesCache[directory][commit][mod_path]['modified']) or len(_availableModulesCache[directory][commit][mod_path]['untracked']) ): changes.append(os.sep.join(mod_path.split(os.sep)[[-2, -3][len(modules_subpath) > 0] :])) if len(changes): if not quiet: printi('-- some modules changes are not part of git --') for mod in tolist(np.unique([x.split(os.sep)[[0, 1][len(modules_subpath) > 0]] for x in changes])): mod_path = directory + os.sep + mod if not os.path.exists(mod_path): raise Exception(mod_path + ' does not exist') tmp = repo.get_module_params(quiet=quiet, path=modules_subpath, key='path', modules=[mod_path]) _availableModulesCache[directory][commit].update(tmp) changed_modules.append(os.path.abspath(mod_path + os.sep + 'OMFITsave.txt')) elif not quiet: printi('-- modules cache is valid --') else: # if same remote/branch then try to update since recorded commit (faster: only need to loop over modules that changed) if ( np.all([k in _availableModulesCache[directory] for k in ['branch', 'remote']]) and len(_availableModulesCache[directory]) == 3 and _availableModulesCache[directory]['remote'] == remote and _availableModulesCache[directory]['branch'] == branch ): printi('-- updating modules cache since last commit --') try: old_commit = [ item for item in list(_availableModulesCache[directory].keys()) if item not in ['branch', 'remote'] ][0] changes = np.unique( [ modules_subpath + '/' + re.sub('^' + modules_subpath + '/', '', x).split(os.sep)[0] for x in repo('diff --name-only %s %s -- %s' % (old_commit, commit, modules_subpath)).split() ] ).tolist() changed_modules = [os.sep.join([repo_dir, x, 'OMFITsave.txt']) for x in changes] _availableModulesCache[directory][commit] = _availableModulesCache[directory][old_commit] del _availableModulesCache[directory][old_commit] _availableModulesCache[directory][commit].update( repo.get_module_params(quiet=quiet, path=modules_subpath, key='path', modules=changed_modules) ) except Exception as _excp: printw('Failed to update available modules cache: ' + repr(_excp)) del _availableModulesCache[directory] else: printi('-- rebuilding modules cache from scratch --') if directory in _availableModulesCache: del _availableModulesCache[directory] # if directory is not in _availableModulesCache then must build everything from scratch if directory not in _availableModulesCache: _availableModulesCache[directory] = {} _availableModulesCache[directory]['branch'] = branch _availableModulesCache[directory]['remote'] = remote _availableModulesCache[directory][commit] = repo.get_module_params( quiet=quiet, path=modules_subpath, key='path' ) updatePersistentCache.append(directory) # delete modules from cache if these are not found on hard drive for mod_path in list(_availableModulesCache[directory][commit].keys()): if not os.path.exists(_availableModulesCache[directory][commit][mod_path]['path']): del _availableModulesCache[directory][commit][mod_path] # update list of modules moduleDict.update(_availableModulesCache[directory][commit]) # do not try 2nd case if 1st one worked break except Exception as _excp: cases_errors.append(_excp) if len(cases_errors) == len(cases): _availableModulesCache[directory].clear() # if the path is not a valid git repository or the data was modified then get modules by hand for mod_path in modules_list_in_dir: if mod_path in moduleDict: continue mod = os.path.split(mod_path)[0] mod_name = os.path.split(mod)[1] if not quiet: printi(mod_name.ljust(20) + ' (untracked)') moduleDict[mod_path] = {} moduleDict[mod_path]['path'] = mod_path moduleDict[mod_path]['untracked'] = mod_path # just for compatibility with data returned by `get_module_params` moduleDict[mod_path]['modified'] = mod_path # just for compatibility with data returned by `get_module_params` moduleDict[mod_path]['edited_by'] = os.environ['USER'] moduleDict[mod_path]['date'] = convertDateFormat(time.time()) moduleDict[mod_path]['commit'] = '' moduleDict[mod_path]['description'] = '' info = OMFITmodule.info(moduleDict[mod_path]['path']) info.pop('date', None) info.pop('edited_by', None) info.pop('commit', None) moduleDict[mod_path].update(info) # save modules info cache if directory in updatePersistentCache or not os.path.exists(directory + os.sep + '.modulesInfoCache'): try: with open(directory + os.sep + '.modulesInfoCache', 'wb') as f: pickle.dump(_availableModulesCache[directory], f, pickle.OMFIT_PROTOCOL) printi('-- Updated persistent git modules cache --') except IOError: pass return moduleDict
[docs] def quit(self, deepClean=False): """ Cleanup current OMFIT session directory `OMFITsessionDir` and PID from `OMFITsessionsDir` Also close all SSH related tunnels and connections :param deepClean: if deepClean is True, then the `OMFITtmpDir` and `OMFITsessionsDir` get deleted """ if OMFITcwd is None: return onlyRunningCopy = OMFIT.onlyRunningCopy(deletePID=True) if onlyRunningCopy: OMFIT.reset_connections() # reset_connections() will also close_mds_connections() else: omfit_classes.omfit_mds.close_mds_connections() try: if os.path.exists(OMFITsessionDir): shutil.rmtree(OMFITsessionDir) except Exception: printw('Some files and directories could not be deleted!') if deepClean: if os.path.exists(OMFITsessionsDir): try: shutil.rmtree(OMFITsessionsDir) os.makedirs(OMFITsessionsDir) except OSError: printw('Some files and directories under %s could not be deleted!' % OMFITsessionsDir) if os.path.exists(OMFITtmpDir): try: shutil.rmtree(OMFITtmpDir) except Exception: printw('Some files and directories under %s could not be deleted!' % OMFITtmpDir) return onlyRunningCopy
[docs] def reset_connections(self, server=None, mds_cache=True): """ Reset connections, stored passwords, and MDS+ cache :param server: only reset SSH connections to this server """ import utils_widgets serverPicker = '' if server is None: printi('Reset OMFIT stored passwords') utils_widgets.stored_passwords.clear() printi('Reset OMFIT server cache') SERVER_cache.clear() if mds_cache: printi('Purge OMFIT MDS+ cache') OMFITmdsCache().purge() else: printi('OMFIT MDS+ cache not purged; use OMFITmdsCache().purge() to purge it') printi('Reset OMFIT SSH connections') printi('Reset OMFIT MDS+ connections') omfit_classes.omfit_mds.close_mds_connections() printi('Reset OMFIT SQL connections') for k in ['MDSserverReachable', 'RDBserverReachable', 'batch_js', 'sshTunnel', 'sysinfo']: OMFITaux[k] = copy.deepcopy(OMFITaux.defaults[k]) else: if server: serverPicker = parse_server(SERVER[server]['server'])[2] printi('Reset OMFIT SSH connections to %s' % serverPicker) sshServices = ['OMFITssh', 'OMFITsshTunnel', 'OMFITsshControlmaster', 'OMFITsshGit'] # Have to create tmpdir since OMFIT tmp directory may have been cleaned up already import tempfile, shutil tmpdir = tempfile.mkdtemp() s = subprocess.Popen("ps xw", stdout=subprocess.PIPE, shell=True, cwd=tmpdir) std_out, std_err = s.communicate() shutil.rmtree(tmpdir) for item in b2s(std_out).split('\n'): service = [s for s in sshServices if re.search(s, item)] if len(service) and re.search(serverPicker, item): pid = int([_f for _f in item.split(' ') if _f][0]) try: printi('kill %s' % (item)) os.kill(int(pid), signal.SIGKILL) except OSError as e: printw('Failed! %s' % repr(e))
[docs] def recentProjects(self, only_read=False): """ This routine keeps track of the recent projects and is also in charge of deleting auto-saves if they do not appear in the project list. :param read_only: only read projects file (don't do any maintenance on it) """ # do not log projects that are in OMFITtmpDir if OMFITtmpDir in OMFIT.filename: return filename_old = OMFITsettingsDir + os.sep + 'OMFITrecentProjects.txt' filename = OMFITsettingsDir + os.sep + 'OMFITprojects.txt' autoSaveLinkDir = os.path.abspath(OMFIT['MainSettings']['SETUP']['projectsDir'].rstrip('/\\')) + os.sep + 'auto-save' + os.sep if not os.path.exists(filename) and os.path.exists(filename_old): shutil.copyfile(filename_old, filename) # read the projects projects = SortedDict() if os.path.exists(filename): with open(filename, 'r') as f: lines = f.readlines() for k, line in enumerate(lines): line = line.strip() tmp = line.split('{') if len(tmp) == 1: path = tmp[0].strip() opts = {} else: path = tmp[0].strip() opts = eval('{' + '{'.join(tmp[1:])) if os.path.exists(path): projects[path] = opts # add extra info (eg. file size) for item in list(projects.keys()): if 'size' not in projects[item] and os.path.exists(item) and item.endswith('.zip') and zipfile.is_zipfile(item): projects[item]['size'] = os.stat(item).st_size if only_read: return projects # this switch will be set to True if the recent projects file needs to be updated doWrite = False # delete `Self-Destruct` projects that have not been accessed in more than 30 days for item in list(projects.keys()): if projects[item].get('type', '') == 'Self-Destruct' and os.stat(item).st_atime < (time.time() - 30 * 86400): if os.path.isdir(item): shutil.rmtree(item) else: os.remove(item) del projects[item] doWrite = True # add projects which for some reason were not already there at the bottom of the list if os.path.exists(os.path.abspath(evalExpr(OMFIT['MainSettings']['SETUP']['projectsDir']))): for item in glob.glob(os.path.abspath(evalExpr(OMFIT['MainSettings']['SETUP']['projectsDir'])) + os.sep + '*'): if os.path.isdir(item): if os.path.exists(item + os.sep + 'OMFITsave.txt'): item = item + os.sep + 'OMFITsave.txt' else: continue if item not in projects: projects[item] = {} doWrite = True # try to read prj_options for the projects which do not have them for item in list(projects.keys()): if 'device' not in projects[item]: try: tmp = OMFITproject.info(item) except Exception as _excp: printe('%s looks like an invalid OMFIT project' % (item)) del projects[item] continue tmp.setdefault('notes', '') for opt in OMFIT.prj_options_choices: if opt in tmp: projects[item][opt] = tmp[opt] doWrite = True # if the current project has been saved put it at the top of the list if ( OMFIT.filename is not None and len(OMFIT.filename) and autoSaveLinkDir not in OMFIT.filename and OMFITautosaveDir not in OMFIT.filename ): item = os.path.abspath(OMFIT.filename) if item in list(projects.keys()): del projects[item] projects.insert(0, item, copy.deepcopy(OMFIT.prj_options)) if item.endswith('.zip') and zipfile.is_zipfile(item): projects[item]['size'] = os.stat(item).st_size elif item.endswith('OMFITsave.txt'): projects[item]['size'] = size_of_dir(os.path.split(item)[0]) doWrite = True # do not store console output, commands, and namespace in OMFITprojects.txt for item in list(projects.keys()): for opt in ['console', 'commands', 'namespace']: if opt in projects[item]: del projects[item][opt] doWrite = True # only store modules locations for item in list(projects.keys()): if isinstance(projects[item]['modules'], dict): projects[item]['modules'] = list(projects[item]['modules'].keys()) # write OMFITprojects.txt~ then move to OMFITprojects.txt if successful # this is to have as much of an atomic operation as possible if doWrite: with open(filename_old + '~', 'w') as f_old, open(filename + '~', 'w') as f: for item in list(projects.keys()): f.write('%s %s\n' % (item, projects[item])) f_old.write('%s\n' % (item)) shutil.move(filename + '~', filename) shutil.move(filename_old + '~', filename_old) # delete auto-saves that are older than one week # process one old auto-save at the time ones that were accessed first (deleting large auto-saves can take some time) for dir in [OMFITautosaveDir, autoSaveLinkDir]: # sort auto-saves by time files = glob.glob(dir + os.sep + '*') # delete broken links (otherwise getatime will fail) remove = [] for path in files: if os.path.islink(path) and not os.path.exists(path): os.remove(path) remove.append(path) files = list(set(files).difference(set(remove))) # sort files files.sort(key=lambda x: os.path.getatime(x)) for path in files: # delete auto-saves that have not been accessed in more than one week if not os.path.islink(path) and os.path.isdir(path) and os.stat(path).st_atime < (time.time() - 7 * 86400): shutil.rmtree(path) break # delete one old auto-save at the time # delete broken links if os.path.islink(path) and not os.path.exists(path): os.remove(path) return projects
[docs] def showExecDiag(self): """ display execution diagram """ def printMe(tmp, depth): for child in tmp: filename = re.sub('__console__.py', '', os.path.split(child['filename'])[1]) printi('* ' * (depth + 1) + " %s %3.3fs [%s]" % (filename, child['time'], sizeof_fmt(child['memory']))) printMe(child['child'], depth + 1) printi('\nExecution diagram:\n------------------') from omfit_classes.omfit_python import ExecDiagPtr printMe(ExecDiagPtr, 0)
[docs]class OMFITmainscratch(OMFITtmp): def __init__(self): self.required = list(map(lambda x: os.path.splitext(os.path.basename(x))[0], glob.glob(OMFITsrc + '/framework_guis/*.py')))
[docs] def initialize(self): for item in self.required: self.load_framework_gui(item)
[docs] def load_framework_gui(self, item): dev_acceptable = os.access(OMFITsrc + os.sep + 'framework_guis' + os.sep + item + '.py', os.W_OK) and not os.path.exists( os.sep.join([OMFITsrc, '..', 'public']) ) self[f'__{item}__'] = OMFITpythonGUI( OMFITsrc + os.sep + 'framework_guis' + os.sep + item + '.py', modifyOriginal=OMFIT['MainSettings']['SETUP']['developer_mode_by_default'] & dev_acceptable, )
def __delitem__(self, key): if key in self.required: self.load_framework_gui(key) else: super().__delitem__(key)
# -------------------------------- # Setup the global variable OMFIT # -------------------------------- OMFIT.__class__ = OMFITmaintree # root of the OMFIT tree OMFIT.modifyOriginal = False OMFIT.readOnly = False OMFIT._OMFITkeyName = 'OMFIT' OMFIT.prj_options = {} OMFIT.prj_options_choices = { 'notes': '', 'commands': '', 'commandNames': [], 'namespace': '', 'console': '', 'shot': '', 'time': '', 'device': '', 'modules': '', 'type': ['Self-Destruct', 'Test', 'Working', 'Done', 'VIP'], 'persistent_projectID': False, 'color': OrderedDict( ( ('red', 'red2'), ('orange', 'dark orange'), ('yellow', 'orange'), ('green', 'forest green'), ('blue', 'mediumblue'), ('purple', 'medium orchid'), ('gray', 'gray25'), ) ), } atexit.register(OMFIT.quit) # Register final automatic cleanup # ------------------------------------ # location from root requires knowing what `OMFIT` is # ------------------------------------
[docs]def locationFromRoot(location): """ :param location: tree location :return: tree location from the closest root """ tmp = location location = parseBuildLocation(location) for k in 1 + np.array(list(range(len(location) - 1))): loc = eval('OMFIT' + parseBuildLocation(location[:k])) if loc.__class__ is OMFITmodule: tmp = "root" + parseBuildLocation(location[k:]) return tmp
[docs]def rootLocation(location, *args, **kw): """ :param location: tree location :return: tree location of the root """ if len(args) >= 1: OMFIT = args[0] if len(args) >= 2: kw['rootName'] = args[1] rootName = kw.setdefault('rootName', 'OMFIT') location = parseBuildLocation(location) root = 'OMFIT' for k in 1 + np.array(list(range(len(location) - 1))): loc = eval('OMFIT' + parseBuildLocation(location[:k])) if loc.__class__ is OMFITmodule: root = 'OMFIT' + parseBuildLocation(location[:k]) root = rootName + root[5:] return root
# --------------------- # Import OMFIT APIs # and make it available to users Python scripts and expressions # --------------------- from omfit_classes import OMFITx setattr(omfit_classes.omfit_python, 'OMFITx', OMFITx) setattr(omfit_classes.omfit_base, 'OMFITx', OMFITx) # --------------------- # graphics # --------------------- from omfit_plot import * # --------------------- # update classes # --------------------- omfit_classes.omfit_base._updateTypes() # --------------------- # S3 # ---------------------
[docs]def OMFITobject_fromS3(filename, s3bucket): """ Recovers OMFITobject from S3 and reloads it with the right class and original keyword parameters :return: object loaded from S3 """ s3connection = boto3.resource('s3', **boto_credentials()) obj = s3connection.Object(s3bucket, filename) cls = eval(eval(obj.metadata.get('__class__', "'OMFITobject'"))) orig_filename = eval(obj.metadata.get('__filename__', filename)) tmp = cls(filename, s3bucket=s3bucket, **{k: eval(obj.metadata[k]) for k in obj.metadata if k not in ['__class__', '__filename__']}) if os.path.split(tmp.filename)[1] != orig_filename: shutil.move(tmp.filename, os.path.split(tmp.filename)[0] + os.sep + os.path.split(orig_filename)[1]) tmp.filename = os.path.split(tmp.filename)[0] + os.sep + os.path.split(orig_filename)[1] tmp.originalFilename = tmp.filename tmp.link = tmp.filename return tmp
# --------------------- # backwards compatibility # --------------------- isNone = is_none
[docs]class OMFITfileASCII(OMFITascii): """ Use of this class is deprecated: use OMFITascii instead """
[docs]class OMFIT_Ufile(OMFITuFile): """ Use of this class is deprecated: use OMFITuFile instead """
[docs]class OMFITdict(SortedDict): """ Use of this class is deprecated: use SortedDict instead """
expr_value = evalExpr
[docs]class chebyfit(omfit_classes.utils_fit.fitCH): """ Use of this class is deprecated: use fitCH instead """
[docs]class OMFITdependenceException(RuntimeError): """ Use of this class is deprecated: use RuntimeError instead """
# ------------------ # this version time (follows last version time in startup.py) # ------------------ _filename = OMFITsettingsDir + os.sep + 'OMFITlastRun.txt' if float(thisVersionTime) > float(latestVersionTime): with open(_filename, 'w') as _f: _f.write(str(thisVersionTime)) # -------------------------------------- # load main settings and override / fix # older versions of users main settings # NOTE: after this point on OMFIT['MainSettings'] is available and can be accessed # -------------------------------------- def _clearDeprecatedMainSettings(MainSettings): # keep track if there has been a change updated = False keep = {} # clear if last version is before given time or if `tmpDir` is found within one month after given time # (this is done to manage transition between versions of OMFIT that are already running) if latestVersionTime < 1427916944 or ( 'SETUP' in MainSettings and 'tmpDir' in MainSettings['SETUP'] and thisVersionTime < (1427916944 + 60 * 60 * 24 * 30) ): keep['SETUP'] = {} keep['SETUP']['email'] = '' keep['SETUP']['modulesDir'] = '' keep['SETUP']['editor'] = '' keep['SETUP']['browser'] = '' keep['SETUP']['error_report'] = '' for k in ['Extensions', 'KeyBindings', 'GUIappearance']: if k in MainSettings['SETUP']: keep['SETUP'][k] = copy.deepcopy(MainSettings['SETUP'][k]) keep['EXPERIMENT'] = {} keep['EXPERIMENT']['device'] = '' keep['EXPERIMENT']['shot'] = '' keep['EXPERIMENT']['time'] = '' keep['SERVER'] = {} keep['SERVER']['GA_username'] = '' keep['SERVER']['PPPL_username'] = '' keep['SERVER']['NERSC_username'] = '' keep['SERVER']['UCSD_username'] = '' keep['SERVER']['ITM_username'] = '' keep['SERVER']['MIT_username'] = '' keep['SERVER']['default'] = '' keep['SERVER']['idl'] = '' keep['SERVER']['matlab'] = '' tmp = prune_mask(MainSettings, keep) MainSettings.clear() MainSettings.update(tmp) updated = True if 'SERVER' in MainSettings: # delete deprecated servers and localhost (localhost entry will be re-created) for server in ( ['thea', 'harvest', 'venus', 'venus_tmp', 'venus_scratch', 'hopper', 'zeus', 'lohan', 'philos', 'edison'] + list(['lohan%d' % x for x in range(20)]) + ['localhost'] ): if server in list(MainSettings['SERVER'].keys()): del MainSettings['SERVER'][server] if server != 'localhost': updated = '1. deprecated_server:' + server # delete references to the deprecated servers for item in list(MainSettings['SERVER'].keys()): if ( 'localhost' not in [item, server] and isinstance(MainSettings['SERVER'][item], str) and MainSettings['SERVER'][item] == server ): del MainSettings['SERVER'][item] updated = '2. deprecated_server: (%s,%s)' % (server, item) # delete servers if any of their entries is set to None for server in list(MainSettings['SERVER'].keys()): if isinstance(MainSettings['SERVER'][server], namelist.NamelistName): for item in list(MainSettings['SERVER'][server].keys()): if MainSettings['SERVER'][server][item] == None: del MainSettings['SERVER'][server] updated = 'none_server' break # freeze usernames username_keys = [item for item in MainSettings['SERVER'] if item.endswith('_username')] for username_key in username_keys: # if OMFIT is running at a recognized institution set username to match os.environ['USER'] if OMFITexpression or empty string if ( (isinstance(MainSettings['SERVER'][username_key], OMFITexpression) or not MainSettings['SERVER'][username_key]) and ('institution' in MainSettings['SETUP']) and username_key == (MainSettings['SETUP']['institution'] + '_username') ): MainSettings['SERVER'][username_key] = os.environ['USER'] updated = 'freeze_username_institution' # for all other servers set username to empty string if username is an OMFITexpression elif isinstance(MainSettings['SERVER'][username_key], OMFITexpression): MainSettings['SERVER'][username_key] = '' updated = 'freeze_username_expression' # NFRI renamed themselves to KFE if ( 'KFE_username' in MainSettings['SERVER'] and not MainSettings['SERVER']['KFE_username'] and 'NFRI_username' in MainSettings['SERVER'] and MainSettings['SERVER']['NFRI_username'] ): MainSettings['SERVER']['KFE_username'] = MainSettings['SERVER']['NFRI_username'] # prevent users to change workDir in bad ways if 'SETUP' in MainSettings and 'workDir' in MainSettings['SETUP'] and OMFITcwd not in str(MainSettings['SETUP']['workDir']): MainSettings['SETUP']['workDir'] = OMFITexpression("OMFITcwd+os.sep+'runs'+os.sep") updated = 'workdir' if ( 'SETUP' in MainSettings and 'GUIappearance' in MainSettings['SETUP'] and 'persistent_projectID' in MainSettings['SETUP']['GUIappearance'] ): del MainSettings['SETUP']['GUIappearance']['persistent_projectID'] updated = 'persistent_project_id' if 'EXPERIMENT' in MainSettings and 'runid' in MainSettings['EXPERIMENT'] and MainSettings['EXPERIMENT']['runid'] == None: MainSettings['EXPERIMENT']['runid'] = 'sim1' updated = 'experiment' return updated OMFIT.reset() _updated = _clearDeprecatedMainSettings(OMFIT['MainSettings']) if _updated: OMFIT.addMainSettings(updateUserSettings=True) # set OMFIT repos directory where the users projects are stored # rationale: home directory have quotas, scratch directories get automatically cleaned omfit_classes.utils_base.OMFITreposDir = ( os.path.split(os.path.abspath(evalExpr(OMFIT['MainSettings']['SETUP']['projectsDir'])))[0] + os.sep + '.repos' ) # ------------------------------------ # Initialize OMFIT tree # ------------------------------------ if not np.any([('sphinx' in k and not 'sphinxcontrib' in k) for k in sys.modules]): OMFIT.start() print('Time to load omfit_tree: %g seconds' % (time.time() - _t0)) # ------------------------------------ # Set new modules skeleton # ------------------------------------ if os.path.exists(OMFITsessionDir + os.sep + 'NEW_MODULE_skeleton'): shutil.rmtree(OMFITsessionDir + os.sep + 'NEW_MODULE_skeleton', ignore_errors=True) shutil.copytree( OMFITsrc + os.sep + 'omfit_classes' + os.sep + 'skeleton' + os.sep + 'NEW_MODULE', OMFITsessionDir + os.sep + 'NEW_MODULE_skeleton' ) OMFITaux['moduleSkeletonCache'] = OMFITtree( OMFITsessionDir + os.sep + 'NEW_MODULE_skeleton' + os.sep + 'OMFITsave.txt', quiet=True, modifyOriginal=True, readOnly=True )