Source code for omfit_tree

# file processed by 2to3
from __future__ import print_function, absolute_import

import warnings

import time
_t0 = time.time()

# override sys.exit to prevent external packages to quit the session
import sys, os

_exit = sys.exit
sys.exit = lambda *args, **kw: None

# ensure that any `import omfit.xxx` refers to this installation of OMFIT
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from classes.startup_framework import *

# import utilities
from utils import _available_to_user_math, _available_to_user_util, _available_to_user_plot, _available_to_user_fusion

#---------------------
# classes under the `class` folder and starting with OMFIT
#---------------------
_loaded_omfit_class_files = []

import classes.omfit_base
from classes.omfit_base import *
from classes.omfit_base import _moduleAttrs, itemTagsValues

import classes.omfit_python
from classes.omfit_python import *

_k = 0
for _file in sorted(glob.glob(OMFITsrc + '/classes/omfit_*.py')):
    _python_module = os.path.splitext(os.path.split(_file)[1])[0]
    try:
        exec("import classes." + _python_module, globals(), locals())
        try:
            getattr(classes, _python_module).__all__
        except AttributeError:
            raise OMFITexception('Exception in omfit/%s: OMFIT does not allow modules under omfit/classes'
                                 'not to have an __all__ attribute which explicitly specifies what symbols '
                                 'will be exported when from <module> import * is used on the module' % _python_module)

        # all classes must be available in omfit_base and omfit_python since that's where the OMFITtree is defined
        if _python_module != 'omfit_base':
            for item in getattr(classes, _python_module).__all__:
                setattr(classes.omfit_base, item, getattr(getattr(classes, _python_module), item))

        if _python_module != 'omfit_python':
            for item in getattr(classes, _python_module).__all__:
                setattr(classes.omfit_python, item, getattr(getattr(classes, _python_module), item))

        exec("from classes import " + _python_module, globals(), locals())
        exec("from classes." + _python_module + ' import *', globals(), locals())
        _k += 1
        _loaded_omfit_class_files.append(('%2d)%s' % (_k, _python_module.replace('omfit_',''))).ljust(20))
    except Exception:
        printe('Offending module: ' + _python_module)
        raise
sys.exit = _exit

# backward compatibility of pickled objects
if 'classes.omfit_weblink' in sys.modules:
    sys.modules['classes.omfit_webLink'] = sys.modules['classes.omfit_weblink']

for item in classes.omfit_python.__all__:
    setattr(classes.omfit_base, item, getattr(omfit_python, item))

print('Loaded OMFIT classes:')
while len(_loaded_omfit_class_files):
    print(''.join(_loaded_omfit_class_files[:4]))
    _loaded_omfit_class_files = _loaded_omfit_class_files[4:]

[docs]def reload_python(moduleName): """ This function extends the Python builtin `reload` function to easily reload OMFIT classes >>> reload(classes.omfit_onetwo) :param moduleName: module or module name """ if not isinstance(moduleName, basestring): moduleName = moduleName.__name__ if not (moduleName.startswith('omfit_') or moduleName.startswith('classes.omfit_')): printe('reload_python is meant for class containing modules (in omfit/classes) that start with `omfit_`') printe('Try instead:\n\nreload({0})\nfrom {0} import *'.format(moduleName)) return if moduleName in ['omfit_tree', 'omfit_gui']: printe('Can not reload omfit_tree or omfit_gui') return tmp = {} exec ("import %s" % moduleName, tmp) if sys.version_info < (3, 0): from builtins import reload else: from importlib import reload reload(eval(moduleName)) exec ("from " + moduleName + " import *", tmp) # keep track of reloaded objects to avoid storing them in the persistent OMFITconsoleDict namespace OMFITreloadedDict.update(tmp) # find objects in the OMFIT tree that should be reloaded OMtr = traverse(OMFIT, string='OMFIT', onlyDict=True, skipDynaLoad=True) OMtr_cls = {} for loc in OMtr: if hasattr(eval(loc), '__class__') and eval(loc).__class__.__name__ in list(tmp.keys()): OMtr_cls.setdefault(eval(loc).__class__.__name__, []).append(loc) # select only the items that should be reloaded items = list(tmp.keys()) if hasattr(eval(moduleName), '__all__'): items = eval(moduleName).__all__ if '__builtins__' in items: items.remove('__builtins__') # loop through items to reload updated = {} for item in items: # loop through python modules for mod in list(sys.modules.keys()): if hasattr(sys.modules[mod], '__dict__') and item in sys.modules[mod].__dict__: sys.modules[mod].__dict__[item] = tmp[item] updated.setdefault(mod, []).append(item) # loop through OMFIT objects if item in OMtr_cls: for loc in OMtr_cls[item]: print(loc) eval(loc).__class__ = tmp[item] # update OMFIT data types classes.omfit_base._updateTypes() printi('*' * 20) printi('Updated python modules') printi('*' * 20) pprinti(updated) printi('') printi('*' * 20) printi('Updated OMFIT tree entries') printi('*' * 20) pprinti(OMtr_cls)
classes.omfit_python.reload_python = reload_python #--------------------- # OMFIT main tree #--------------------- _availableModulesCache={}
[docs]class OMFITmaintree(OMFITproject): _save_method='_save_with_info' def __init__(self, filename=''): OMFITproject.__init__(self,filename) self._OMFITparent=None self._OMFITkeyName='' self.prj_options={}
[docs] def start(self,filename=''): self.clear() self.filename=filename self.reset() self.onlyRunningCopy() OMFIT['MainSettings'].sort()
[docs] def projectName(self): if not len(self.filename): return '' if re.findall('OMFITsave.txt',self.filename): return os.path.split(self.filename.split(os.sep+'OMFITsave.txt')[0])[1] else: return os.path.splitext(os.path.split(self.filename)[1])[0]
[docs] def onlyRunningCopy(self, deletePID=False): ''' :param delete: whether to remove PID from list of running OMFIT processes (this should be done only at exit) :return: return True/False wether this is the only running copy of OMFIT on this computer ''' filename=os.sep.join([OMFITsessionsDir,str(os.getpid())]) #OMFITsessionsDir should always exist! -- if not return False if not os.path.exists(OMFITsessionsDir): printe('Something is wrong! OMFITsessionsDir (%s) has been deleted!'%OMFITsessionsDir) return False else: try: # this is in a try/except because # in some extreme circumstances there can be # errors if the system cannot allocate memory if not os.path.exists(filename): open(filename,'w').close() pids=[] for file in glob.glob(os.sep.join([OMFITsessionsDir,'*'])): pid=os.path.split(file)[1] if is_running(pid): pids.append(pid) elif deletePID: try: os.remove(file) except OSError: pass if deletePID and os.path.exists(filename): os.remove(filename) return len(pids)==1 except OSError: return False
[docs] def reset(self): self.zip=True self.clear() OMFITconsoleDict.clear() OMFITscriptsDict.clear() # remove all files under current temporary directory try: shutil.rmtree(OMFITcwd) except Exception: pass if not os.path.exists(OMFITcwd): os.makedirs(OMFITcwd) os.chdir(OMFITcwd) # create main settings self.addMainSettings(restore='user') OMFIT['MainSettings']['EXPERIMENT']['runid']='sim1' self['MainSettings']['SETUP']['version'] = repo_active_branch_commit self['MainSettings']['SETUP']['python_environment']=SortedDict(python_environment()) #set the localhost SERVER.setLocalhost()
[docs] def newProvenanceID(self): self['MainSettings']['EXPERIMENT']['provenanceID']=omfit_hash(utils_base.now("%Y-%m-%d_%H_%M_%S_%f"))
[docs] def newProjectID(self): self['MainSettings']['EXPERIMENT']['projectID']='projectID__' self['MainSettings']['EXPERIMENT']['projectID']+=repo_active_branch_commit+'__' self['MainSettings']['EXPERIMENT']['projectID']+=utils_base.now("%Y-%m-%d_%H_%M_%S_%f") self['MainSettings']['EXPERIMENT']['projectID']=self['MainSettings']['EXPERIMENT']['projectID'].replace(' ','')
[docs] def addMainSettings(self, updateUserSettings=False, restore=''): # read the user namelist file self.userMainSettings = OMFITsettingsDir+os.sep+'MainSettings.txt' self.userMainSettingsNamelistDump = OMFITsettingsDir+os.sep+'MainSettingsNamelistDump.txt' self.userMainSettingsNamelist = OMFITsettingsDir+os.sep+'MainSettingsNamelist.txt' if not os.path.exists(os.path.split(self.userMainSettings)[0]): printe('Created '+os.path.split(self.userMainSettings)[0]+' directory') os.makedirs(os.path.split(self.userMainSettings)[0]) if not os.path.exists(self.userMainSettings): printe('Created '+self.userMainSettings) open(self.userMainSettings,'w').close() updateUserSettings=True if not os.path.exists(self.userMainSettingsNamelist): printe('Created '+self.userMainSettingsNamelist) open(self.userMainSettingsNamelist,'w').close() updateUserSettings=True # keep project options for k in list(OMFIT.prj_options_choices.keys()): if k not in self.prj_options: if k=='persistent_projectID': self.prj_options[k]=False else: self.prj_options[k]='' # force restore of skel main settings if necessary self.apply_bindings() if 'MainSettings' not in self or restore!='': #skeleton settings skelMainSettings = OMFITsrc+os.sep+'skeleton'+os.sep+'skeletonMainSettings.txt' self.tmpSkel=OMFITtree(skelMainSettings,quiet=True) if platform.system()=='Darwin': skelMainSettings = OMFITsrc+os.sep+'skeleton'+os.sep+'skeletonMainSettingsNamelistOSX.txt' else: skelMainSettings = OMFITsrc+os.sep+'skeleton'+os.sep+'skeletonMainSettingsNamelistUNIX.txt' self.tmpSkel['MainSettings'].recursiveUpdate(namelist.NamelistFile(skelMainSettings),overwrite=True) #institution settings filename=os.sep.join([OMFITsrc,'..','institution']) if os.path.exists(filename): tmp = namelist.NamelistFile(filename) else: institution_files=glob.glob(os.sep.join([OMFITsrc,'..','institutions',''])+'*') institution_files.append(None) for filename in institution_files: tmp = namelist.NamelistFile(filename) if 'SETUP' in tmp and 'stats_file' in tmp['SETUP'] and os.path.exists(tmp['SETUP']['stats_file']): break self.tmpSkel['MainSettings'].recursiveUpdate(tmp,overwrite=True) #user settings self.tmpUser=OMFITtree(self.userMainSettings,quiet=True) if 'MainSettings' not in self: self['MainSettings']=OMFITmainSettings() self['MainSettings'].filename=OMFITcwd+os.sep+os.path.split(self.userMainSettingsNamelist)[1] if not len(self.tmpUser): restore='skel' updateUserSettings=True #The main settings are built starting from the skeleton, the user local preference in ~/.OMFIT/ and the edits that the users have made if restore=='': self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) def f_traverse(me): for kid in list(me.keys()): if isinstance(me[kid],dict): f_traverse(me[kid]) if re.match(hide_ptrn,kid): del me[kid] f_traverse(self['MainSettings']) self.add_bindings_to_main_settings() elif restore.startswith('diff_'): restore=restore[5:] tmp=OMFITmainSettings() if restore=='skel': tmp.recursiveUpdate(self.tmpSkel['MainSettings']) elif restore=='user' and 'MainSettings' in self.tmpUser: tmp.recursiveUpdate(self.tmpSkel['MainSettings']) tmp.recursiveUpdate(self.tmpUser['MainSettings'],overwrite=True) elif restore=='S3': tmp.recursiveUpdate(self.tmpSkel['MainSettings']) tmp.recursiveUpdate(self.tmpUser['MainSettings'],overwrite=True) tmp1=OMFITobject_fromS3(tmp['SETUP']['email']+"_"+'MainSettings.txt',s3bucket='omfit') tmp2=OMFITobject_fromS3(tmp['SETUP']['email']+"_"+'MainSettingsNamelist.txt',s3bucket='omfit') shutil.copy2(tmp2.filename,os.path.split(tmp1.filename)[0]+os.sep+os.path.split(tmp2.filename)[1]) tmp0=OMFITtree(tmp1.filename,quiet=True) tmp.recursiveUpdate(tmp0['MainSettings'],overwrite=True) printi('Loaded MainSettings from the cloud') diffTreeGUI(OMFIT['MainSettings'], tmp) else: self['MainSettings'].clear() if restore=='skel': self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) elif restore=='user' and 'MainSettings' in self.tmpUser: self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) self['MainSettings'].recursiveUpdate(self.tmpUser['MainSettings'],overwrite=True) elif restore=='S3': self['MainSettings'].recursiveUpdate(self.tmpSkel['MainSettings']) self['MainSettings'].recursiveUpdate(self.tmpUser['MainSettings'],overwrite=True) tmp1=OMFITobject_fromS3(self['MainSettings']['SETUP']['email']+"_"+'MainSettings.txt',s3bucket='omfit') tmp2=OMFITobject_fromS3(self['MainSettings']['SETUP']['email']+"_"+'MainSettingsNamelist.txt',s3bucket='omfit') shutil.copy2(tmp2.filename,os.path.split(tmp1.filename)[0]+os.sep+os.path.split(tmp2.filename)[1]) tmp=OMFITtree(tmp1.filename,quiet=True) self['MainSettings'].recursiveUpdate(tmp['MainSettings'],overwrite=True) printi('Restored MainSettings from the cloud') self.apply_bindings() #generate unique projectID if that's not already there if 'projectID' not in self['MainSettings']['EXPERIMENT']: self.newProjectID() #generate unique provenanceID if that's not already there if 'provenanceID' not in self['MainSettings']['EXPERIMENT']: self.newProvenanceID() #make sure auto-save is not more often than 15 minutes try: self['MainSettings']['SETUP']['autosave_minutes']=int(self['MainSettings']['SETUP']['autosave_minutes']) if self['MainSettings']['SETUP']['autosave_minutes']<15: raise(OMFITexception('Auto-save time must be >= than 15 minutes')) except Exception as _excp: printe('Error in setting Auto-save time\n'+repr(_excp)) self['MainSettings']['SETUP']['autosave_minutes']=15 #if default_tunnel is None, then set it based on what servers the user has access if self['MainSettings']['SERVER']['default_tunnel'] is None: favorite_tunnels=['cybele','portal','cmodws','shenma','itm_gateway','iter_login','cfetr'] default_tunnel='cybele' if os.path.exists(os.environ['HOME']+'/.ssh/known_hosts'): with open(os.environ['HOME']+'/.ssh/known_hosts','r') as f: lines=[_f for _f in f.read().strip().split('\n') if _f] known_tunnels=[] for line in lines: try: for server in line.split()[0].split(','): try: SERVER[server] if SERVER(server) in favorite_tunnels: known_tunnels.append(SERVER(server)) except Exception: pass except IndexError: pass for tunnel in favorite_tunnels: if tunnel in known_tunnels: default_tunnel=tunnel break self['MainSettings']['SERVER']['default_tunnel']=default_tunnel if updateUserSettings: #find differences of current MainSettings with respect to skeleton tmpU=OMFITtree() tmpU['MainSettings']=copy.deepcopy(OMFIT['MainSettings']) ptrn=self.tmpSkel.pretty_diff(tmpU) # do not store projectID, provenanceID, or runid for k in ['projectID','provenanceID','runid']: if k in list(ptrn['MainSettings']['EXPERIMENT'].keys()): del ptrn['MainSettings']['EXPERIMENT'][k] # keep only differences tmpU=prune_mask(tmpU,ptrn) # remove entries that are deprecated _clearDeprecatedMainSettings(tmpU['MainSettings']) #remove entries that are not in the skeleton for item in list(tmpU['MainSettings'].keys()): if item in list(tmpU['MainSettings'].keys()) and item not in self.tmpSkel['MainSettings']: del tmpU['MainSettings'][item] if item in list(tmpU['MainSettings'].keys()) and item != 'SERVER': for subitem in list(tmpU['MainSettings'][item].keys()): if subitem in list(tmpU['MainSettings'][item].keys()) and subitem not in self.tmpSkel['MainSettings'][item] and subitem!='KeyBindings': del tmpU['MainSettings'][item][subitem] elif subitem=='KeyBindings': for desc,event in list(tmpU['MainSettings'][item][subitem].items()): if desc in global_event_bindings.default_desc_event and global_event_bindings.default_desc_event[desc]==event: del tmpU['MainSettings'][item][subitem][desc] #save only differences (if there are any) if len(tmpU): tmpU._save(filename=self.userMainSettings,only="['MainSettings']",onlyOMFITsave=True) global OMFITexpressionsReturnNone tmpExp=OMFITexpressionsReturnNone OMFITexpressionsReturnNone=True try: tmpU['MainSettings'].saveas(self.userMainSettingsNamelist) finally: OMFITexpressionsReturnNone=tmpExp self.tmpUser=tmpU printi(self.userMainSettings+' has been updated') if updateUserSettings=='S3': OMFITascii(OMFITsettingsDir+os.sep+'MainSettings.txt').deploy(self['MainSettings']['SETUP']['email']+"_"+'MainSettings.txt',s3bucket='omfit') OMFITascii(OMFITsettingsDir+os.sep+'MainSettingsNamelist.txt').deploy(self['MainSettings']['SETUP']['email']+"_"+'MainSettingsNamelist.txt',s3bucket='omfit') printi('Saved user MainSettings in the cloud') # these entries must always be in the OMFIT tree if 'scratch' not in self or not isinstance(self['scratch'],OMFITtmp): self['scratch']=OMFITtmp() for item in ['preferencesGUI', 'styleGUI', 'moduleSetupGUI', 'namelistGUI', 'imasActorGUI', 'imasGUI', 'latexGUI', 'developerModeGUI']: if '__%s__'%item not in self['scratch']: dev_acceptable = os.access(OMFITsrc+os.sep+'skeleton'+os.sep+item+'.py',os.W_OK) and not os.path.exists(os.sep.join([OMFITsrc, '..', 'public'])) self['scratch']['__%s__'%item]=OMFITpythonGUI(OMFITsrc+os.sep+'skeleton'+os.sep+item+'.py', modifyOriginal=OMFIT['MainSettings']['SETUP']['developer_mode_by_default'] & dev_acceptable) self['commandBox']=OMFITconsoleDict self['scriptsRun']=OMFITscriptsDict self['shotBookmarks']=OMFITshotBookmarks if updateUserSettings or not os.path.exists(self.userMainSettingsNamelistDump): #save a version of MainSettings in plain text self['MainSettings'].deploy(self.userMainSettingsNamelistDump)
[docs] def add_bindings_to_main_settings(self): ''' Take the descriptions and events from global_events_bindings and insert them in self['MainSettings']['SETUP']['KeyBindings'][desc] = <event> ''' printd('Calling add_bindings_to_main_settings',level=2,topic='keybindings') if 'KeyBindings' not in self['MainSettings']['SETUP']: self['MainSettings']['SETUP']['KeyBindings'] = namelist.NamelistName() for di,d in enumerate(global_event_bindings.descs): self['MainSettings']['SETUP']['KeyBindings'][d.replace(' ','_')] = global_event_bindings.events[di]
[docs] def apply_bindings(self): ''' Take the descriptions and events from self['MainSettings']['SETUP']['KeyBindings'] and use them to update the global_event_bindings ''' if ('MainSettings' not in self) or ('SETUP' not in self['MainSettings']) or ('KeyBindings' not in self['MainSettings']['SETUP']): printd("No key bindings in MainSettings['SETUP']['KeyBindings'] -> unable to update",topic='keybindings') return for d,e in list(self['MainSettings']['SETUP']['KeyBindings'].items()): try: global_event_bindings.set(d.replace('_',' '),e) except ValueError: del self['MainSettings']['SETUP']['KeyBindings'][d] self.add_bindings_to_main_settings()
[docs] def save(self, quiet=False, skip_save_errors=False): """ Writes the content of the OMFIT tree to the filesystem using the same filename and zip options of the last saveas :param quiet: whether to print save progress to screen :param skip_save_errors: skip errors when saving objects """ self.filename = self._save_with_info(self.filename, zip=self.zip, quiet=quiet, skip_save_errors=skip_save_errors) return self.filename
def _save_with_info(self, filename, zip=False, quiet=False, updateExistingDir=False, skip_save_errors=False): ''' Wrapper function aroun the OMFITtree._save method which handles the prj_options dictionary that is only of the main OMFIT tree ''' #this method should have the same signature of the ._save() method if filename: #by default the first time the projects directory is created, #assign permissions so that it is readeable but not writeable #by other users. Users can always override these default permissions #once the directory is created. projpath=os.path.split(os.path.split(filename)[0])[0] if not os.path.exists(projpath): #create projects directory os.makedirs(projpath) #by default, remove group and others write permission for perm in [stat.S_IWGRP,stat.S_IWOTH]: st_mode=os.lstat(projpath).st_mode os.chmod(projpath, st_mode & ~perm) #by default, add group and others read and browse permission for perm in [stat.S_IRGRP,stat.S_IROTH,stat.S_IXGRP,stat.S_IXOTH]: st_mode=os.lstat(projpath).st_mode os.chmod(projpath, st_mode | perm) #update list of shots/times from mainSettings and modules settings modules=self.moduleDict() for k in ['device','shot','time']: self.prj_options[k]=[] for k in ['device','shot','shots','time','times']: self.prj_options[k.rstrip('s')].extend(tolist(evalExpr(self['MainSettings']['EXPERIMENT'][k]))) try: for module in list(modules.keys()): mod=eval('OMFIT'+module) self.prj_options.setdefault(k.rstrip('s'),[]).extend(tolist(evalExpr(mod['SETTINGS']['EXPERIMENT'][k]))) self.prj_options[k.rstrip('s')]=list([_f for _f in self.prj_options[k.rstrip('s')] if _f]) except Exception as _excp: printe('Error accessing `%s` of module `OMFIT%s` : %s'%(k,module,repr(_excp))) for k in ['device','shot','time']: self.prj_options[k]=unique(list([_f for _f in self.prj_options[k.rstrip('s')] if _f])).tolist() #if the GUI is open update the GUI elements info if OMFITaux['GUI']: commands=[] for k in range(len(OMFITaux['GUI'].command)): tmp=OMFITaux['GUI'].command[k].get(1.0,tk.END).strip() if len(tmp): commands.append(OMFITaux['GUI'].command[k].get(1.0,tk.END)) if not len(commands): commands.append('') OMFIT.prj_options['commands']=commands OMFIT.prj_options['namespace'] = OMFITaux['GUI'].commandBoxNamespace OMFIT.prj_options['notes']=OMFITaux['GUI'].notes.get(1.0,tk.END).strip() OMFIT.prj_options['console']=OMFITaux['console'].get() OMFIT.prj_options.setdefault('color','') if OMFIT.prj_options['color'] not in list(OMFIT.prj_options_choices['color'].keys()): OMFIT.prj_options['color']='' if OMFIT.prj_options['type'] not in OMFIT.prj_options_choices['type']: OMFIT.prj_options['type']='' try: OMFITaux['hardLinks']=True self['__GUISAVE__']=self.prj_options cb = OMFITtree() for k, v in enumerate(self['__GUISAVE__']['commands']): if not v.strip(): continue try: cb['%d.py'%(k+1)] = OMFITascii(filename='%d.py'%(k+1),fromString=v.rstrip()) except Exception as _excp: printe(_excp) if list(cb.keys()): self['__COMMANDBOX__'] = cb orig_version = self['MainSettings']['SETUP']['version'] orig_environ = self['MainSettings']['SETUP']['python_environment'] self['MainSettings']['SETUP']['version'] = repo_active_branch_commit self['MainSettings']['SETUP']['python_environment']=SortedDict(python_environment()) return self._save(filename, zip=zip, quiet=quiet, skipStorage=False, updateExistingDir=updateExistingDir,skip_save_errors=skip_save_errors) except Exception: self['MainSettings']['SETUP']['version'] = orig_version self['MainSettings']['SETUP']['python_environment'] = orig_environ raise finally: if '__GUISAVE__' in self: del self['__GUISAVE__'] if '__COMMANDBOX__' in self: del self['__COMMANDBOX__'] OMFITaux['hardLinks']=False else: raise(IOError('Last saved directory was not defined'))
[docs] def saveas(self, filename, zip=None, quiet=False, skip_save_errors=False): """ Writes the content of the OMFIT tree to the filesystem and permanently changes the name of the project :param filename: project filename to save to :param zip: whether the save should occur as a zip file :param quiet: whether to print save progress to screen :param skip_save_errors: skip errors when saving objects """ oldZip=self.zip oldFilename=self.filename oldPrj_options=self.prj_options if zip is not None: self.zip=zip self.filename=filename try: return self.save(quiet=quiet, skip_save_errors=skip_save_errors) except Exception: self.zip=oldZip self.filename=oldFilename self.prj_options=oldPrj_options raise
[docs] def loadModule(self, filename, location=None, withSubmodules=True, availableModulesList=None, checkLicense=True, developerMode=None, depth=0): """ Load a module in OMFIT :param filename: * full path to the module to be loaded * if just the module name is provided, this will be loaded from the pubblic modules * remote/branch:module format will load a module from a specific git remote and branch * module:remote/branch format will load a module from a specific git remote and branch :param location: string with the location where to place the module in the OMFIT tree :param withSubmodules: load submodules or not :param availableModulesList: list of available modules generated by `OMFIT.availableModules()` If this list is not passed, then the availableModulesList is generated internally :param checkLicense: Check license files at load :param developerMode: Load module with developer mode option (ie. scripts loaded as modifyOriginal) if None then default behavior is set by OMFIT['MainSettings']['SETUP']['developer_mode_by_default'] Note: public OMFIT installation cannot be loaded in developer mode :param depth: parameter used internally by for keeping track of the recursion depth :return: instance of the loaded module """ if os.sep not in filename or ':' in filename: # if there is a semicolumn in the filename, then a specific remote/branch is been requested if ':' in filename: work_repo = repo.clone() remote, branch = filter(lambda x: '/' in x, filename.split(':'))[0].split('/') filename = filter(lambda x: '/' not in x, filename.split(':'))[0] work_repo.switch_branch(branch, remote) filename = os.sep.join([work_repo.git_dir, 'modules', filename, 'OMFITsave.txt']) else: # if it's just a module name, load it from the first of the OMFITmodule directories filename = os.sep.join([OMFITmodule.directories()[0], filename, 'OMFITsave.txt']) filename = os.path.abspath(filename) # get list of available modules if availableModulesList is None: availableModulesList = self.availableModules(quiet=True, same_path_as=filename) # handle developerMode switch OMFITsrc = os.path.split(os.path.split(os.path.split(filename)[0])[0])[0]+os.sep+'omfit' if os.path.exists(os.sep.join([OMFITsrc, '..', 'public'])): developerMode = False if developerMode is None: developerMode = OMFIT['MainSettings']['SETUP']['developer_mode_by_default'] # load the module moduleName = OMFITmodule.info(filename).get('ID', 'unknown_module') if location is None: location = "self['" + moduleName + "']" tmp = OMFITmodule(filename, developerMode=developerMode) # remove the module filename as there is no point of keeping it once it's loaded from the repo # OMFITtree (and derived classes) filenames are used for saving them external to the project tmp.filename = '' # if loading as a submodule, then place at the top loc = parseLocation(location) if isinstance(eval(buildLocation(loc[:-1])), OMFITmodule): eval(buildLocation(loc[:-1])).insert(0, loc[-1], tmp) else: eval(buildLocation(loc[:-1]))[loc[-1]] = tmp # add extra commit info to MODULE if filename in availableModulesList: for k in _moduleAttrs: if k in availableModulesList[filename]: tmp['SETTINGS']['MODULE'][k] = availableModulesList[filename][k] # fill in missing infos (e.g. when starting a new module) if not tmp['SETTINGS']['MODULE']['commit']: tmp['SETTINGS']['MODULE']['commit'] = repo.get_hash('HEAD') if tmp['SETTINGS']['MODULE']['contact'] is None: tmp['SETTINGS']['MODULE']['contact'] = [] if not len(tmp['SETTINGS']['MODULE']['contact']) and OMFIT['MainSettings']['SETUP']['email']: tmp['SETTINGS']['MODULE']['contact'].append(OMFIT['MainSettings']['SETUP']['email']) # check lincense if checkLicense and 'license' in eval(location): license = eval(location)['license'] email_dict = {} with open(license.filename, 'r') as f: for k in f.readlines(): k = str(k) if 'OMFIT_license_email_notify' in k: notify=[x.strip() for x in re.sub(r'OMFIT_license_email_notify\:(.*)',r'\1',k).strip().split(',')] email_dict = { 'email_address': notify, 'email_prompt': 'Please describe planned research using ' + moduleName, 'fromm': self['MainSettings']['SETUP']['email']} break try: OMFITlicenses[moduleName] = License(moduleName, license.filename, email_dict=email_dict, rootGUI=OMFITaux['rootGUI']) OMFITlicenses[moduleName].check() except LicenseException: exec ('del ' + location, globals(), locals()) raise except tk.TclError: # If there is no DISPLAY, allow access pass except IndexError: # If gui element fails to load, allow access pass # load the submodules (will set DEPENDENCIES, SETTINGS['EXPERIMENT'], and honor LIB['OMFITlib_startup'] scripts) if withSubmodules: # find the submodules moduleDict = eval(location).moduleDict() # save the dependencies depsDict = {} for subMod in list(moduleDict.keys()): depsDict[subMod] = eval(location + subMod)['SETTINGS']['DEPENDENCIES'] # load the submodules for subMod in list(moduleDict.keys()): found = False for avMod in availableModulesList: if moduleDict[subMod]['ID'] == availableModulesList[avMod]['ID']: OMFIT.loadModule(availableModulesList[avMod]['path'], location + subMod, withSubmodules=True, availableModulesList=availableModulesList, checkLicense=checkLicense, developerMode=developerMode, depth=depth + 1) found = True break if not found: raise IOError('Module %s is a submodule of %s, and it could not found in %s' % (moduleDict[subMod]['ID'], moduleName, os.path.split(os.path.split(filename)[0])[0])) # restore the dependencies for subMod in list(moduleDict.keys()): for avMod in availableModulesList: if moduleDict[subMod]['ID'] == availableModulesList[avMod]['ID']: eval(location + subMod)['SETTINGS']['DEPENDENCIES'].update(depsDict[subMod]) break # freeze the root['SETTINGS']['EXPERIMENT'] entries if this is a top-level import that is not a submodule if depth == 0 and 'SETTINGS' in eval(location) and 'EXPERIMENT' in eval(location)['SETTINGS']: freeze = True for tmpName in reversed(traverseLocation(location)[:-1]): if eval(tmpName).__class__ is OMFITmodule and tmpName != 'OMFIT': freeze = False if freeze: for item in list(OMFITaux['moduleSkeletonCache']['SETTINGS']['EXPERIMENT'].keys()): eval(location)['SETTINGS']['EXPERIMENT'][item] = evalExpr(eval(location)['SETTINGS']['EXPERIMENT'][item]) # execute startup script (the OMFITmodulePath variable is setup to point to the OMFITsave.txt file of the module that is being loaded) if 'LIB' in eval(location) and 'OMFITlib_startup' in eval(location)['LIB']: try: printi("Running startup script %s['LIB']['OMFITlib_startup']" % location) eval(location)['LIB']['OMFITlib_startup'].runNoGUI(OMFITmodulePath=filename) except Exception as _excp: printe("Error in " + location + "['LIB']['OMFITlib_startup'] : " + repr(_excp)) # return the module return eval(location)
[docs] def load(self, filename, persistent_projectID=False): ''' loads an OMFITproject in OMFIT :param filename: filename of the project to load (if `-1` then the most recent project is loaded) :param persistent_projectID: whether the projectID should remain the same ''' try: #open the last project if filename=='-1': projects=OMFIT.recentProjects() if len(projects): filename=list(projects.keys())[0] persistent_projectID=projects[filename].get('persistent_projectID',False) #keep aside user 'SETUP','SERVER' tmp={} for item in ['SETUP','SERVER']: if item in self['MainSettings']: tmp[item]=copy.deepcopy(self['MainSettings'][item]) self.filename=os.path.abspath(filename) oldDir=os.getcwd() TMPnoCopyToCWD=OMFITaux['noCopyToCWD'] OMFITaux['noCopyToCWD']=False try: self._load(filename) finally: os.chdir(oldDir) OMFITaux['noCopyToCWD']=TMPnoCopyToCWD if zipfile.is_zipfile(self.filename): self.zip = True projectName = os.path.split(self.filename)[1] else: self.zip = False projectName = os.path.split(os.path.split(self.filename)[0])[1] # for older projects we calculate provenanceID based on institution and projectName if 'provenanceID' not in self['MainSettings']['EXPERIMENT']: self['MainSettings']['EXPERIMENT']['provenanceID'] = omfit_hash(self['MainSettings']['SETUP']['institution'] + '_' + projectName) # SETUP is overwritten (but not version nor python_environment) project_version=self['MainSettings']['SETUP'].get('version','?') project_environ=self['MainSettings']['SETUP'].get('python_environment',{}) self['MainSettings']['SETUP']=tmp['SETUP'] self['MainSettings']['SETUP']['version']=project_version self['MainSettings']['SETUP']['python_environment']=project_environ # SERVER is updated self['MainSettings'].setdefault('SERVER',NamelistName()).update(tmp.get('SERVER',{})) #load project options self.prj_options.clear() tmp=OMFITproject.info(self.filename) for k in OMFIT.prj_options_choices: self.prj_options[k]=tmp.get(k,'') for whereSave in ['__GUISAVE__','__COMMANDBOX__']: if whereSave in OMFIT: del OMFIT[whereSave] if self.prj_options['color'] not in list(self.prj_options_choices['color'].keys()): self.prj_options['color']='' if self.prj_options['type'] not in self.prj_options_choices['type']: self.prj_options['type']='' if 'namespace' not in self.prj_options or not self.prj_options['namespace']: self.prj_options['namespace'] = 'OMFIT' #update projectID self.prj_options['persistent_projectID']=persistent_projectID if not persistent_projectID: self.newProjectID() #keep MainSettings in check self.updateMainSettings() self['MainSettings'].sort() #set the localhost SERVER.setLocalhost() #remote projects are loaded in OMFITtmpDir #reset the filename to force users to decide how/where to save it if OMFITtmpDir in self.filename: self.filename='' except Exception: self.start() raise
[docs] def updateCWD(self): global OMFITcwd self._save(filename=OMFITcwd+os.sep+'OMFITsave.txt', skipStorage=False)
[docs] def updateMainSettings(self): self.addMainSettings() self.keyOrder.remove('scratch') self.keyOrder.append('scratch') self.keyOrder.remove('commandBox') self.keyOrder.append('commandBox') self.keyOrder.remove('scriptsRun') self.keyOrder.append('scriptsRun') self.keyOrder.remove('shotBookmarks') self.keyOrder.append('shotBookmarks') self.keyOrder.remove('MainSettings') self.keyOrder.append('MainSettings')
[docs] def saveMainSettingsSkeleton(self): ''' This utility function updates the MainSettingsSkeleton for the current OMFIT installation ''' self['MainSettings'].sort() tmp=copy.deepcopy(self['MainSettings']) self.addMainSettings(restore='skel') self['MainSettings'].sort() try: self['MainSettings'].filename=self['MainSettings']['SETUP']['workDir']+'../'+'skeletonMainSettingsNamelist.txt' self['MainSettings']['SETUP']['version']='?' self['MainSettings']['SETUP']['python_environment']={} del self['MainSettings']['EXPERIMENT']['projectID'] del self['MainSettings']['EXPERIMENT']['provenanceID'] self['MainSettings']['SETUP']['browser']=None self['MainSettings']['SETUP']['Extensions'].clear() for k in tmp['SERVER']: if k not in self['MainSettings']['SERVER']: self['MainSettings']['SERVER'][k]=tmp['SERVER'][k] if isinstance(tmp['SERVER'][k],namelist.NamelistName): self['MainSettings']['SERVER'][k].update(tmp['SERVER'][k]) self['MainSettings']['SERVER']['localhost']=namelist.NamelistName() self['MainSettings']['SERVER']['localhost']['workDir']=OMFITexpression("OMFITcwd+os.sep+'remote_runs'+os.sep") self['MainSettings']['SERVER']['localhost']['server']='localhost' self['MainSettings']['SERVER']['localhost']['tunnel']='' self['MainSettings']['SERVER']['localhost']['idl']='' self['MainSettings']['SERVER']['localhost']['matlab']='' self['MainSettings']['SERVER']['default_tunnel']=None finally: self['MainSettings'].sort() self._save(filename=OMFITsrc+os.sep+'skeleton'+os.sep+'skeletonMainSettings.txt',only="['MainSettings']",updateExistingDir=True) self['MainSettings']=tmp
[docs] def availableModules(self=None, quiet=False, directories=None, same_path_as=None, force=False): """ Index available OMFIT modules in directories :param quiet: verbosity :param directories: list of directories to index. If `None` this is taken from OMFIT['MainSettings']['SETUP']['modulesDir'] :param same_path_as: sample OMFITsave.txt path to set directory to index :param force: force rebuild of .modulesInfoCache file :return: This method returns a dictionary with the available OMFIT modules. Each element in the dictionary is a dictionary itself with the details of the available modules. """ if directories is None: directories=OMFITmodule.directories() else: directories=tolist(directories) if same_path_as is None: pass elif 'OMFITsave.txt' in same_path_as and os.path.exists(same_path_as): directories=[os.path.split(os.path.split(same_path_as)[0])[0]] else: raise(IOError('%s file is not a valid OMFITsave.txt file'%same_path_as)) updatePersistentCache = [] moduleDict = {} for directory in directories: # the user inputs the directory where the modules are if not quiet: tmp = 'Modules directory: ' + directory printi('=' * len(tmp)) printi(tmp) printi('=' * len(tmp)) if not force and os.path.exists(directory + os.sep + '.modulesInfoCache'): try: with open(directory + os.sep + '.modulesInfoCache', 'rb') as f: _availableModulesCache[directory] = pickle.load(f) except Exception: _availableModulesCache[directory] = {} printw('-- Failed to load persistent git modules cache --') else: _availableModulesCache[directory] = {} # modules in directory modules_list_in_dir = sorted([os.path.abspath(x) for x in glob.glob(os.sep.join([directory, '*', 'OMFITsave.txt']))], key=lambda x:x.lower()) # try handling git repository of modules twice # first as if git root directory parent directory of modules directory # second as if git root directory is modules directory cases_errors = [] cases = [] if os.path.split(directory)[1] == 'modules': cases.append([os.path.split(directory)[0], 'modules']) cases.append([directory, '']) for repo_dir, modules_subpath in cases: try: # try to access as a git repository repo = OMFITgit(repo_dir) commit = repo("log -1 --pretty='%H'") branch = repo('rev-parse --abbrev-ref HEAD') remote = repo.get_branches().get(branch, {'remote': ''})['remote'] changed_modules = [] # if same remote/branch and commit if all([k in _availableModulesCache[directory] for k in ['branch', 'remote', commit]]) and _availableModulesCache[directory]['remote'] == remote and _availableModulesCache[directory]['branch'] == branch: changes = list([_f for _f in repo("ls-files -m --others --exclude-standard " + directory).split('\n') if _f]) for mod_path in modules_list_in_dir: if mod_path not in _availableModulesCache[directory][commit] or len(_availableModulesCache[directory][commit][mod_path]['modified']) or len(_availableModulesCache[directory][commit][mod_path]['untracked']): changes.append(os.sep.join(mod_path.split(os.sep)[[-2, -3][len(modules_subpath) > 0]:])) if len(changes): if not quiet: printi('-- some modules changes are not part of git --') for mod in tolist(unique([x.split(os.sep)[[0, 1][len(modules_subpath) > 0]] for x in changes])): mod_path = directory + os.sep + mod if not os.path.exists(mod_path): raise (Exception(mod_path + ' does not exist')) tmp = repo.get_module_params(quiet=False, path=modules_subpath, key='path', modules=[mod_path]) _availableModulesCache[directory][commit].update(tmp) changed_modules.append(os.path.abspath(mod_path + os.sep + 'OMFITsave.txt')) elif not quiet: printi('-- modules cache is valid --') else: # if same remote/branch then try to update since recorded commit (faster: only need to loop over modules that changed) if all([k in _availableModulesCache[directory] for k in ['branch', 'remote']]) and len(_availableModulesCache[directory]) == 3 and _availableModulesCache[directory]['remote'] == remote and _availableModulesCache[directory]['branch'] == branch: printi('-- updating modules cache since last commit --') try: old_commit = [item for item in list(_availableModulesCache[directory].keys()) if item not in ['branch', 'remote']][0] changes = unique([modules_subpath + '/' + re.sub('^' + modules_subpath + '/', '', x).split(os.sep)[0] for x in repo('diff --name-only %s %s -- %s' % (old_commit, commit, modules_subpath)).split()]).tolist() changed_modules = [os.sep.join([repo_dir, x, 'OMFITsave.txt']) for x in changes] _availableModulesCache[directory][commit] = _availableModulesCache[directory][old_commit] del _availableModulesCache[directory][old_commit] _availableModulesCache[directory][commit].update(repo.get_module_params(quiet=quiet, path=modules_subpath, key='path', modules=changed_modules)) except Exception as _excp: printw('Failed to update available modules cache: ' + repr(_excp)) del _availableModulesCache[directory] else: printi('-- rebuilding modules cache from scratch --') if directory in _availableModulesCache: del _availableModulesCache[directory] # if directory is not in _availableModulesCache then must build everything from scratch if directory not in _availableModulesCache: _availableModulesCache[directory] = {} _availableModulesCache[directory]['branch'] = branch _availableModulesCache[directory]['remote'] = remote _availableModulesCache[directory][commit] = repo.get_module_params(quiet=quiet, path=modules_subpath, key='path') updatePersistentCache.append(directory) # delete modules from cache if these are not found on hard drive for mod_path in list(_availableModulesCache[directory][commit].keys()): if not os.path.exists(_availableModulesCache[directory][commit][mod_path]['path']): del _availableModulesCache[directory][commit][mod_path] # update list of modules moduleDict.update(_availableModulesCache[directory][commit]) # do not try 2nd case if 1st one worked break except Exception as _excp: cases_errors.append(_excp) if len(cases_errors) == len(cases): _availableModulesCache[directory].clear() printe('Error handling modules') for item in cases_errors: printe('* ' + repr(item)) # if the path is not a valid git repository or the data was modified then get modules by hand for mod_path in modules_list_in_dir: if mod_path in moduleDict: continue mod = os.path.split(mod_path)[0] mod_name = os.path.split(mod)[1] if not quiet: printi(mod_name.ljust(20) + ' (untracked)') moduleDict[mod_path] = {} moduleDict[mod_path]['path'] = mod_path moduleDict[mod_path]['untracked'] = mod_path # just for compatibility with data returned by `get_module_params` moduleDict[mod_path]['modified'] = mod_path # just for compatibility with data returned by `get_module_params` moduleDict[mod_path]['edited_by'] = os.environ['USER'] moduleDict[mod_path]['date'] = convertDateFormat(time.time()) moduleDict[mod_path]['commit'] = '' moduleDict[mod_path]['description'] = '' info = OMFITmodule.info(moduleDict[mod_path]['path']) info.pop('date', None) info.pop('edited_by', None) info.pop('commit', None) moduleDict[mod_path].update(info) # save modules info cache if directory in updatePersistentCache or not os.path.exists(directory + os.sep + '.modulesInfoCache'): try: with open(directory + os.sep + '.modulesInfoCache', 'wb') as f: pickle.dump(_availableModulesCache[directory], f, pickle.OMFIT_PROTOCOL) printi('-- Updated persistent git modules cache --') except IOError: pass return moduleDict
[docs] def quit(self, deepClean=False): ''' Cleanup current OMFIT session directory `OMFITsessionDir` and PID from `OMFITsessionsDir` Also close all SSH related tunnels and connections :param deepClean: if deepClean is True, then the `OMFITtmpDir` and `OMFITsessionsDir` get deleted ''' if OMFITcwd is None: return try: if os.path.exists(OMFITsessionDir): shutil.rmtree(OMFITsessionDir) except Exception: printw('Some files and directories could not be deleted!') onlyRunningCopy = OMFIT.onlyRunningCopy(deletePID=True) if onlyRunningCopy: OMFIT.reset_connections() if deepClean: if os.path.exists(OMFITsessionsDir): try: shutil.rmtree(OMFITsessionsDir) except OSError: printw('Some files and directories under %s could not be deleted!' % OMFITsessionsDir) if os.path.exists(OMFITtmpDir): try: shutil.rmtree(OMFITtmpDir) except Exception: printw('Some files and directories under %s could not be deleted!' % OMFITtmpDir) return onlyRunningCopy
[docs] def reset_connections(self): printi('Reset OMFIT MDS+ connections') printi('Reset OMFIT SQL connections') printi('Reset OMFIT SSH tunnel connections') printi('Reset OMFIT stored passwords') import utils_widgets utils_widgets.stored_passwords.clear() for k in ['MDSserverReachable', 'RDBserverReachable', 'batch_js', 'sshTunnel', 'sysinfo']: OMFITaux[k] = copy.deepcopy(OMFITaux.defaults[k]) for k in list(classes.omfit_mds._OMFITmdsConnectionBaseObjects.keys()): classes.omfit_mds._OMFITmdsConnectionBaseObjects[k].clear() del classes.omfit_mds._OMFITmdsConnectionBaseObjects[k] sshServices = ['OMFITssh', 'OMFITsshTunnel', 'OMFITsshControlmaster', 'OMFITsshGit'] s = subprocess.Popen(["ps", "xw"], stdout=subprocess.PIPE) std_out, std_err = s.communicate() for item in b2s(std_out).split('\n'): service = [service for service in sshServices if re.search(service, item)] if len(service): pid = int([_f for _f in item.split(' ') if _f][0]) try: printi('PID %d : killed %s' % (pid, service[0])) os.kill(int(pid), signal.SIGKILL) except OSError as e: printw('PID %d : killed %s : %s' % (pid, service[0], repr(e)))
[docs] def recentProjects(self, only_read=False): """ This routine keeps track of the recent projects and is also in charge of deleting auto-saves if they do not appear in the project list. :param read_only: only read projects file (don't do any maintenance on it) """ #do not log projects that are in OMFITtmpDir if OMFITtmpDir in OMFIT.filename: return filename_old=OMFITsettingsDir+os.sep+'OMFITrecentProjects.txt' filename=OMFITsettingsDir+os.sep+'OMFITprojects.txt' autoSaveLinkDir=os.path.abspath(OMFIT['MainSettings']['SETUP']['projectsDir'].rstrip('/\\'))+os.sep+'auto-save'+os.sep if not os.path.exists(filename) and os.path.exists(filename_old): shutil.copyfile(filename_old,filename) # read the projects projects=SortedDict() if os.path.exists(filename): with open(filename,'r') as f: lines=f.readlines() for k,line in enumerate(lines): line=line.strip() tmp=line.split('{') if len(tmp)==1: path=tmp[0].strip() opts={} else: path=tmp[0].strip() opts=eval('{'+'{'.join(tmp[1:])) if os.path.exists(path): projects[path]=opts # add extra info (eg. file size) for item in list(projects.keys()): if 'size' not in projects[item] and os.path.exists(item) and item.endswith('.zip') and zipfile.is_zipfile(item): projects[item]['size']=os.stat(item).st_size if only_read: return projects # this switch will be set to True if the recent projects file needs to be updated doWrite=False # add projects which for some reason were not already there at the bottom of the list if os.path.exists(os.path.abspath(evalExpr(OMFIT['MainSettings']['SETUP']['projectsDir']))): for item in glob.glob(os.path.abspath(evalExpr(OMFIT['MainSettings']['SETUP']['projectsDir']))+os.sep+'*'): if os.path.isdir(item): if os.path.exists(item+os.sep+'OMFITsave.txt'): item=item+os.sep+'OMFITsave.txt' else: continue if item not in projects: projects[item]={} doWrite=True # try to read prj_options for the projects which do not have them for item in list(projects.keys()): if 'device' not in projects[item]: try: tmp=OMFITproject.info(item) except Exception as _excp: printe('%s looks like an invalid OMFIT project'%(item)) del projects[item] continue tmp.setdefault('notes','') for k in OMFIT.prj_options_choices: if k in tmp: projects[item][k]=tmp[k] doWrite=True # if the current project has been saved put it at the top of the list if OMFIT.filename is not None and len(OMFIT.filename) and autoSaveLinkDir not in OMFIT.filename and OMFITautosaveDir not in OMFIT.filename: item=os.path.abspath(OMFIT.filename) if item in list(projects.keys()): del projects[item] projects.insert(0,item,copy.deepcopy(OMFIT.prj_options)) if item.endswith('.zip') and zipfile.is_zipfile(item): projects[item]['size']=os.stat(item).st_size elif item.endswith('OMFITsave.txt'): projects[item]['size'] = size_of_dir(os.path.split(item)[0]) doWrite=True #do not store console output, commands, and namespace in OMFITprojects.txt for item in list(projects.keys()): for k in ['console','commands','namespace']: if k in projects[item]: del projects[item][k] doWrite=True # write OMFITprojects.txt~ then move to OMFITprojects.txt if successful # this is to have as much of an atomic opreration as possible if doWrite: with open(filename_old+'~','w') as f_old, open(filename+'~','w') as f: for item in list(projects.keys()): f.write('%s %s\n'%(item,projects[item])) f_old.write('%s\n'%(item)) shutil.move(filename+'~',filename) shutil.move(filename_old+'~',filename_old) #cleanup auto-saves that are older than one week #delete one old auto-save at the time, starting from oldest one for dir in [OMFITautosaveDir,autoSaveLinkDir]: #sort auto-saves by time files=glob.glob(dir+os.sep+'*') #delete broken links (otherwise getmtime will fail) remove=[] for path in files: if os.path.islink(path) and not os.path.exists(path): os.remove(path) remove.append(path) printw('Removed missing auto-save: '+path) files=list(set(files).difference(set(remove))) #sort files files.sort(key=lambda x: os.path.getmtime(x)) for path in files: #delete auto-saves that are older than one week if not os.path.islink(path) and os.path.isdir(path) and os.stat(path).st_mtime < time.time() - 7 * 86400: shutil.rmtree(path) break #delete one old auto-save at the time #delete broken links if os.path.islink(path) and not os.path.exists(path): os.remove(path) return projects
# -------------------------------- # Setup the global variable OMFIT # -------------------------------- OMFIT.__class__ = OMFITmaintree # root of the OMFIT tree OMFIT.modifyOriginal = False OMFIT.readOnly = False OMFIT._OMFITkeyName = 'OMFIT' OMFIT.prj_options = {} OMFIT.prj_options_choices = {'notes': '', 'commands': '', 'namespace': '', 'console': '', 'shot': '', 'time': '', 'device': '', 'modules': '', 'type': ['Test', 'Working', 'Done', 'VIP'], 'persistent_projectID': False, 'color': OrderedDict((('red', 'red2'), ('orange', 'dark orange'), ('yellow', 'orange'), ('green', 'forest green'), ('blue', 'mediumblue'), ('purple', 'medium orchid'), ('gray', 'gray25'))) } atexit.register(OMFIT.quit) # Register final automatic cleanup #------------------------------------ #location from root requires knowing what `OMFIT` is #------------------------------------
[docs]def locationFromRoot(location): """ :param location: tree location :return: tree location from the closest root """ tmp=location location=parseBuildLocation(location) for k in 1+numpy.array(list(range(len(location)-1))): loc=eval('OMFIT'+parseBuildLocation(location[:k])) if loc.__class__ is OMFITmodule: tmp="root"+parseBuildLocation(location[k:]) return tmp
[docs]def rootLocation(location,*args,**kw): ''' :param location: tree location :return: tree location of the root ''' if len(args)>=1: OMFIT=args[0] if len(args)>=2: kw['rootName']=args[1] rootName=kw.setdefault('rootName','OMFIT') location=parseBuildLocation(location) root='OMFIT' for k in 1+numpy.array(list(range(len(location)-1))): loc=eval('OMFIT'+parseBuildLocation(location[:k])) if loc.__class__ is OMFITmodule: root='OMFIT'+parseBuildLocation(location[:k]) root=rootName+root[5:] return root
#--------------------- # Import OMFIT APIs # and make it available to users Python scripts and expressions #--------------------- from classes import OMFITx setattr(classes.omfit_python, 'OMFITx', OMFITx) setattr(classes.omfit_base, 'OMFITx', OMFITx) #--------------------- # graphics #--------------------- from omfit_plot import * #--------------------- # update classes #--------------------- classes.omfit_base._updateTypes() #--------------------- # S3 #---------------------
[docs]def OMFITobject_fromS3(filename,s3bucket): ''' Recovers OMFITobject from S3 and reloads it with the right class and original keyword parameters :return: object loaded from S3 ''' s3connection = boto3.resource('s3',**boto_credentials()) obj=s3connection.Object(s3bucket, filename) cls=eval(eval(obj.metadata.get('__class__',"'OMFITobject'"))) orig_filename=eval(obj.metadata.get('__filename__',filename)) tmp=cls(filename, s3bucket=s3bucket, **{k:eval(obj.metadata[k]) for k in obj.metadata if k not in ['__class__','__filename__']}) if os.path.split(tmp.filename)[1]!=orig_filename: shutil.move(tmp.filename,os.path.split(tmp.filename)[0]+os.sep+os.path.split(orig_filename)[1]) tmp.filename=os.path.split(tmp.filename)[0]+os.sep+os.path.split(orig_filename)[1] tmp.originalFilename=tmp.filename tmp.link=tmp.filename return tmp
#--------------------- # backwards compatibility #--------------------- isNone = is_none
[docs]class OMFITfileASCII(OMFITascii): """ Use of this class is deprecated: use OMFITascii instead """
[docs]class OMFIT_Ufile(OMFITuFile): """ Use of this class is deprecated: use OMFITuFile instead """
[docs]class OMFITdict(SortedDict): """ Use of this class is deprecated: use SortedDict instead """
expr_value = evalExpr
[docs]class chebyfit(classes.utils_fit.fitCH): """ Use of this class is deprecated: use fitCH instead """
[docs]class OMFITdependenceException(RuntimeError): """ Use of this class is deprecated: use RuntimeError instead """
#------------------ # this version time (follows last version time in startup.py) #------------------ _filename=OMFITsettingsDir+os.sep+'OMFITlastRun.txt' if float(thisVersionTime)>float(latestVersionTime): with open(_filename,'w') as _f: _f.write(str(thisVersionTime)) #-------------------------------------- # load main settings and override / fix # older versions of users main settings # NOTE: after this point on OMFIT['MainSettings'] is available and can be accessed #-------------------------------------- def _clearDeprecatedMainSettings(MainSettings): #keep track if there has been a change updated=False keep={} #clear if last version is before given time or if `tmpDir` is found within one month after given time #(this is done to manage transition between versions of OMFIT that are already running) if latestVersionTime<1427916944 or ('SETUP' in MainSettings and 'tmpDir' in MainSettings['SETUP'] and thisVersionTime<(1427916944+60*60*24*30)): keep['SETUP']={} keep['SETUP']['email']='' keep['SETUP']['modulesDir']='' keep['SETUP']['editor']='' keep['SETUP']['browser']='' keep['SETUP']['error_report']='' for k in ['Extensions','KeyBindings','GUIappearance']: if k in MainSettings['SETUP']: keep['SETUP'][k]=copy.deepcopy(MainSettings['SETUP'][k]) keep['EXPERIMENT']={} keep['EXPERIMENT']['device']='' keep['EXPERIMENT']['shot']='' keep['EXPERIMENT']['time']='' keep['SERVER']={} keep['SERVER']['GA_username']='' keep['SERVER']['PPPL_username']='' keep['SERVER']['NERSC_username']='' keep['SERVER']['UCSD_username']='' keep['SERVER']['ITM_username']='' keep['SERVER']['MIT_username']='' keep['SERVER']['default']='' keep['SERVER']['idl']='' keep['SERVER']['matlab']='' tmp=prune_mask(MainSettings,keep) MainSettings.clear() MainSettings.update( tmp ) updated=True if 'SERVER' in MainSettings: #delete deprecated servers and localhost (localhost entry will be re-created) for server in ['thea','harvest','venus','venus_tmp','venus_scratch','hopper','zeus','lohan','philos','edison']+list(['lohan%d'%x for x in range(20)])+['localhost']: if server in list(MainSettings['SERVER'].keys()): del MainSettings['SERVER'][server] if server!='localhost': updated='1. deprecated_server:'+server #delete references to the deprecated servers for item in list(MainSettings['SERVER'].keys()): if 'localhost' not in [item,server] and isinstance(MainSettings['SERVER'][item],basestring) and MainSettings['SERVER'][item]==server: del MainSettings['SERVER'][item] updated='2. deprecated_server: (%s,%s)'%(server,item) #delete servers if any of their entries is set to None for server in list(MainSettings['SERVER'].keys()): if isinstance(MainSettings['SERVER'][server],namelist.NamelistName): for item in list(MainSettings['SERVER'][server].keys()): if MainSettings['SERVER'][server][item]==None: del MainSettings['SERVER'][server] updated='none_server' break #freeze usernames username_keys=[item for item in MainSettings['SERVER'] if item.endswith('_username')] for username_key in username_keys: #if OMFIT is running at a recognized institution set username to match os.environ['USER'] if OMFITexpression or empty string if (isinstance(MainSettings['SERVER'][username_key],OMFITexpression) or not MainSettings['SERVER'][username_key]) and ('institution' in MainSettings['SETUP']) and username_key==(MainSettings['SETUP']['institution']+'_username'): MainSettings['SERVER'][username_key]=os.environ['USER'] updated='freeze_username_institution' #for all other servers set username to empty string if username is an OMFITexpression elif isinstance(MainSettings['SERVER'][username_key],OMFITexpression): MainSettings['SERVER'][username_key]='' updated='freeze_username_expression' #prevent users to change workDir in bad ways if 'SETUP' in MainSettings and 'workDir' in MainSettings['SETUP'] and OMFITcwd not in str(MainSettings['SETUP']['workDir']): MainSettings['SETUP']['workDir']=OMFITexpression("OMFITcwd+os.sep+'runs'+os.sep") updated='workdir' if 'SETUP' in MainSettings and 'GUIappearance' in MainSettings['SETUP'] and 'persistent_projectID' in MainSettings['SETUP']['GUIappearance']: del MainSettings['SETUP']['GUIappearance']['persistent_projectID'] updated='persistent_project_id' if 'EXPERIMENT' in MainSettings and 'runid' in MainSettings['EXPERIMENT'] and MainSettings['EXPERIMENT']['runid']==None: MainSettings['EXPERIMENT']['runid']='sim1' updated='experiment' return updated OMFIT.addMainSettings(restore='user') _updated=_clearDeprecatedMainSettings(OMFIT['MainSettings']) if _updated: OMFIT.addMainSettings(updateUserSettings=True) # set OMFIT repos directory where the users projects are stored # rationale: home directory have quotas, scratch directories get automatically cleaned classes.utils_base.OMFITreposDir=os.path.split(os.path.abspath(evalExpr(OMFIT['MainSettings']['SETUP']['projectsDir'])))[0]+os.sep+'.repos' #------------------------------------ # Initialize OMFIT tree #------------------------------------ if not any([('sphinx' in k and not 'sphinxcontrib' in k) for k in sys.modules]): OMFIT.start() print('Time to load omfit_tree: %g seconds'%(time.time()-_t0)) #------------------------------------ # Set new modules skeleton #------------------------------------ if os.path.exists(OMFITsessionDir+os.sep+'NEW_MODULE_skeleton'): shutil.rmtree(OMFITsessionDir+os.sep+'NEW_MODULE_skeleton', ignore_errors=True) shutil.copytree(OMFITsrc+os.sep+'skeleton'+os.sep+'NEW_MODULE',OMFITsessionDir+os.sep+'NEW_MODULE_skeleton') OMFITaux['moduleSkeletonCache']=OMFITtree(OMFITsessionDir+os.sep+'NEW_MODULE_skeleton'+os.sep+'OMFITsave.txt',quiet=True,modifyOriginal=True,readOnly=True)