Changeset 1059 for trunk/src/series.m
- Timestamp:
- Nov 29, 2018, 5:44:44 PM (6 years ago)
- File:
-
- 1 edited
-
trunk/src/series.m (modified) (8 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/src/series.m
r1051 r1059 1 2 3 4 1 %'series': master function associated to the GUI series.m for analysis field series 5 2 %------------------------------------------------------------------------ … … 212 209 end 213 210 %get the list of previous campaigns in the upper bar menu Open campaign 214 if isfield(h,'MenuCampaign')215 for ifile=1:min(length(h.MenuCampaign),5)216 set(handles.(['MenuCampaign_' num2str(ifile)]),'Label',h.MenuCampaign{ifile});217 end218 end211 % if isfield(h,'MenuCampaign') 212 % for ifile=1:min(length(h.MenuCampaign),5) 213 % set(handles.(['MenuCampaign_' num2str(ifile)]),'Label',h.MenuCampaign{ifile}); 214 % end 215 % end 219 216 %get the menu of actions 220 217 if isfield(h,'ActionListUser') && iscell(h.ActionListUser) && isfield(h,'ActionPathListUser') && iscell(h.ActionPathListUser) … … 498 495 end 499 496 end 500 501 OutPut=browse_data(oldfile,'on','on'); % open the GUI browse_data to get select a campaign dir, experiment and device 502 NbLines=numel(OutPut.Experiment)*numel(OutPut.DataSeries); 503 icount=0; 504 for iexp=1:numel(OutPut.Experiment) 505 for idevice=1:numel(OutPut.DataSeries) 506 icount=icount+1; 507 InputTable{icount,1}=fullfile(OutPut.Campaign,OutPut.Experiment{iexp}); 508 InputTable{icount,2}=OutPut.DataSeries{idevice}; 509 if isempty(InputTable{icount,3}) 510 if icount>1 511 InputTable{icount,3}=InputTable{icount-1,3}; 512 else 513 InputTable{icount,3}=''; 514 end 515 end 516 if isempty(InputTable{icount,4}) 517 if icount>1 518 InputTable{icount,4}=InputTable{icount-1,4}; 519 else 520 InputTable{icount,4}=''; 521 end 522 end 523 if isempty(InputTable{icount,5}) 524 if icount>1 525 InputTable{icount,5}=InputTable{icount-1,5}; 526 else 527 InputTable{icount,5}=''; 528 end 529 end 530 end 531 end 532 if size(InputTable,1)>icount 533 InputTable(icount+1:size(InputTable,1),:)=[]; 534 end 497 InputTable{1,1}='...'; 535 498 set(handles.InputTable,'Data',InputTable) 536 REFRESH_Callback(hObject, eventdata, handles) 499 browse_data(oldfile,'on','on'); % open the GUI browse_data to get select a campaign dir, experiment and device 500 % NbLines=numel(OutPut.Experiment)*numel(OutPut.DataSeries); 501 % icount=0; 502 % for iexp=1:numel(OutPut.Experiment) 503 % for idevice=1:numel(OutPut.DataSeries) 504 % icount=icount+1; 505 % InputTable{icount,1}=fullfile(OutPut.Campaign,OutPut.Experiment{iexp}); 506 % InputTable{icount,2}=OutPut.DataSeries{idevice}; 507 % if isempty(InputTable{icount,3}) 508 % if icount>1 509 % InputTable{icount,3}=InputTable{icount-1,3}; 510 % else 511 % InputTable{icount,3}=''; 512 % end 513 % end 514 % if isempty(InputTable{icount,4}) 515 % if icount>1 516 % InputTable{icount,4}=InputTable{icount-1,4}; 517 % else 518 % InputTable{icount,4}=''; 519 % end 520 % end 521 % if isempty(InputTable{icount,5}) 522 % if icount>1 523 % InputTable{icount,5}=InputTable{icount-1,5}; 524 % else 525 % InputTable{icount,5}=''; 526 % end 527 % end 528 % end 529 % end 530 % if size(InputTable,1)>icount 531 % InputTable(icount+1:size(InputTable,1),:)=[]; 532 % end 533 %REFRESH_Callback(hObject, eventdata, handles) 537 534 538 535 % -------------------------------------------------------------------- 539 function MenuCampaign_Callback(hObject, eventdata, handles)540 % --------------------------------------------------------------------541 542 OutPut=browse_data(get(hObject,'Label'),'on','on'); % open the GUI browse_data to get select a campaign dir, experiment and device543 if ~isfield(OutPut,'Campaign')544 return545 end546 NbLines=numel(OutPut.Experiment)*numel(OutPut.DataSeries);547 icount=0;548 InputTable=get(handles.InputTable,'Data');549 for iexp=1:numel(OutPut.Experiment)550 for idevice=1:numel(OutPut.DataSeries)551 icount=icount+1;552 InputTable{icount,1}=fullfile(OutPut.Campaign,OutPut.Experiment{iexp});553 InputTable{icount,2}=OutPut.DataSeries{idevice};554 if isempty(InputTable{icount,3})555 if icount>1556 InputTable{icount,3}=InputTable{icount-1,3};557 else558 InputTable{icount,3}='';559 end560 end561 if isempty(InputTable{icount,4})562 if icount>1563 InputTable{icount,4}=InputTable{icount-1,4};564 else565 InputTable{icount,4}='';566 end567 end568 if isempty(InputTable{icount,5})569 if icount>1570 InputTable{icount,5}=InputTable{icount-1,5};571 else572 InputTable{icount,5}='';573 end574 end575 end576 end577 if size(InputTable,1)>icount578 InputTable(icount+1:size(InputTable,1),:)=[];579 end580 set(handles.InputTable,'Data',InputTable)581 REFRESH_Callback(hObject, eventdata, handles)536 % function MenuCampaign_Callback(hObject, eventdata, handles) 537 % % -------------------------------------------------------------------- 538 % 539 % OutPut=browse_data(get(hObject,'Label'),'on','on'); % open the GUI browse_data to get select a campaign dir, experiment and device 540 % if ~isfield(OutPut,'Campaign') 541 % return 542 % end 543 % NbLines=numel(OutPut.Experiment)*numel(OutPut.DataSeries); 544 % icount=0; 545 % InputTable=get(handles.InputTable,'Data'); 546 % for iexp=1:numel(OutPut.Experiment) 547 % for idevice=1:numel(OutPut.DataSeries) 548 % icount=icount+1; 549 % InputTable{icount,1}=fullfile(OutPut.Campaign,OutPut.Experiment{iexp}); 550 % InputTable{icount,2}=OutPut.DataSeries{idevice}; 551 % if isempty(InputTable{icount,3}) 552 % if icount>1 553 % InputTable{icount,3}=InputTable{icount-1,3}; 554 % else 555 % InputTable{icount,3}=''; 556 % end 557 % end 558 % if isempty(InputTable{icount,4}) 559 % if icount>1 560 % InputTable{icount,4}=InputTable{icount-1,4}; 561 % else 562 % InputTable{icount,4}=''; 563 % end 564 % end 565 % if isempty(InputTable{icount,5}) 566 % if icount>1 567 % InputTable{icount,5}=InputTable{icount-1,5}; 568 % else 569 % InputTable{icount,5}=''; 570 % end 571 % end 572 % end 573 % end 574 % if size(InputTable,1)>icount 575 % InputTable(icount+1:size(InputTable,1),:)=[]; 576 % end 577 % set(handles.InputTable,'Data',InputTable) 578 % REFRESH_Callback(hObject, eventdata, handles) 582 579 583 580 … … 742 739 %% enable other menus and uicontrols 743 740 % set(handles.MenuOpenCampaign,'Enable','on') 744 set(handles.MenuCampaign_1,'Enable','on')745 set(handles.MenuCampaign_2,'Enable','on')746 set(handles.MenuCampaign_3,'Enable','on')747 set(handles.MenuCampaign_4,'Enable','on')748 set(handles.MenuCampaign_5,'Enable','on')741 % set(handles.MenuCampaign_1,'Enable','on') 742 % set(handles.MenuCampaign_2,'Enable','on') 743 % set(handles.MenuCampaign_3,'Enable','on') 744 % set(handles.MenuCampaign_4,'Enable','on') 745 % set(handles.MenuCampaign_5,'Enable','on') 749 746 set(handles.RUN, 'Enable','On') 750 747 set(handles.RUN,'BackgroundColor',[1 0 0])% set RUN button to red … … 1594 1591 end 1595 1592 1596 %% create the output data directory if needed 1597 OutputDir=''; 1598 if isfield(Param,'OutputSubDir') 1599 SubDirOut=[get(handles.OutputSubDir,'String') Param.OutputDirExt]; 1600 SubDirOutNew=SubDirOut; 1601 detect=exist(fullfile(Param.InputTable{1,1},SubDirOutNew),'dir'); % test if the dir already exist 1602 check_create=1; % need to create the result directory by default 1603 CheckOverwrite=1; 1604 if isfield(Param,'CheckOverwrite') 1605 CheckOverwrite=Param.CheckOverwrite; 1606 end 1607 while detect 1608 if CheckOverwrite 1609 comment=', possibly overwrite previous data'; 1593 %% Look for prcessing on multiple experiments set by the GUI browse_data 1594 NbExp=1; 1595 ListExp=Param.InputTable(1,1); 1596 1597 if get(handles.Replicate,'Value') 1598 hh=findobj(allchild(0),'Tag','browse_data'); 1599 BrowseData=guidata(hh); 1600 SourceDir=get(BrowseData.SourceDir,'String'); 1601 ListExp=get(BrowseData.ListExperiments,'String'); 1602 ListExp=ListExp(get(BrowseData.ListExperiments,'Value')); 1603 NbExp=numel(ListExp) % number of experiments set possibly by the GUI browse_data, =1 otherwise 1604 for ilist=1:NbExp 1605 ListExp{ilist}=regexprep(ListExp{ilist},'+',''); 1606 ListExp{ilist}= [SourceDir ListExp{ilist}]; 1607 end 1608 end 1609 1610 %%%%%%%%%%%%%%%%%%% LOOP ON EXPERIMENTS POSSIBLY SET BY THE GUI browse_data, NbExp=1 otherwise %%%%%%%%% 1611 for iexp=1:NbExp 1612 Param.InputTable{1,1}=ListExp{iexp}; 1613 set(handles.InputTable,'Data',Param.InputTable) 1614 [xx,ExpName]=fileparts(ListExp{iexp}); 1615 Param.IndexRange.first_i=str2num(get(handles.num_first_i,'String'));%reset the firrst_i and last_i for multiple experiments, modified by the splitting into NbProcess 1616 Param.IndexRange.last_i=str2num(get(handles.num_last_i,'String')); 1617 %% create the output data directory if needed 1618 OutputDir=''; 1619 answer=''; 1620 if isfield(Param,'OutputSubDir') 1621 SubDirOut=[get(handles.OutputSubDir,'String') Param.OutputDirExt]; 1622 SubDirOutNew=SubDirOut; 1623 detect=exist(fullfile(Param.InputTable{1,1},SubDirOutNew),'dir'); % test if the dir already exist 1624 check_create=1; % need to create the result directory by default 1625 CheckOverwrite=1; 1626 if isfield(Param,'CheckOverwrite') 1627 CheckOverwrite=Param.CheckOverwrite; 1628 end 1629 while detect 1630 if CheckOverwrite 1631 comment=', possibly overwrite previous data'; 1632 else 1633 comment=', will complement existing result files (no overwriting)'; 1634 end 1635 answer=msgbox_uvmat('INPUT_Y-N-Cancel',['use existing ouput directory: ' fullfile(Param.InputTable{1,1},SubDirOutNew) comment]); 1636 if strcmp(answer,'Cancel') 1637 break 1638 elseif strcmp(answer,'Yes') 1639 detect=0; 1640 check_create=0; 1641 else 1642 r=regexp(SubDirOutNew,'(?<root>.*\D)(?<num1>\d+)$','names'); % detect whether name ends by a number 1643 if isempty(r) 1644 r(1).root=[SubDirOutNew '_']; 1645 r(1).num1='0'; 1646 end 1647 SubDirOutNew=[r(1).root num2str(str2num(r(1).num1)+1)]; % increment the index by 1 or put 1 1648 detect=exist(fullfile(Param.InputTable{1,1},SubDirOutNew),'dir'); % test if the dir already exists 1649 check_create=1; 1650 end 1651 end 1652 if strcmp(answer,'Cancel') 1653 continue 1654 end 1655 Param.OutputDirExt=regexprep(SubDirOutNew,Param.OutputSubDir,''); 1656 Param.OutputRootFile=Param.InputTable{1,3}; % the first sorted RootFile taken for output 1657 set(handles.OutputDirExt,'String',Param.OutputDirExt) 1658 OutputDir=fullfile(Param.InputTable{1,1},[Param.OutputSubDir Param.OutputDirExt]); % full name (with path) of output directory 1659 if check_create % create output directory if it does not exist 1660 [tild,msg1]=mkdir(OutputDir); 1661 if ~strcmp(msg1,'') 1662 errormsg=['cannot create ' OutputDir ': ' msg1]; % error message for directory creation 1663 return 1664 end 1665 end 1666 1667 elseif isfield(Param,'ActionInput')&&isfield(Param.ActionInput,'LogPath')% custom definition of the output dir 1668 OutputDir=Param.ActionInput.LogPath; 1669 end 1670 DirXml=fullfile(OutputDir,'0_XML'); 1671 if ~exist(DirXml,'dir') 1672 [~,msg1]=mkdir(DirXml); 1673 if ~strcmp(msg1,'') 1674 errormsg=['cannot create ' DirXml ': ' msg1]; % error message for directory creation 1675 return 1676 end 1677 end 1678 OutputNomType=nomtype2pair(Param.InputTable{1,4}); % nomenclature for output files 1679 1680 %% get the set of reference input field indices 1681 first_i=1; % first i index to process 1682 last_i=1; % last i index to process 1683 incr_i=1; % increment step in i index 1684 first_j=1; % first j index to process 1685 last_j=1; % last j index to process 1686 incr_j=1; % increment step in j index 1687 if isfield(Param.IndexRange,'first_i') 1688 first_i=Param.IndexRange.first_i; 1689 incr_i=Param.IndexRange.incr_i; 1690 last_i=Param.IndexRange.last_i; 1691 end 1692 if isfield(Param.IndexRange,'incr_j') 1693 first_j=Param.IndexRange.first_j; 1694 last_j=Param.IndexRange.last_j; 1695 incr_j=Param.IndexRange.incr_j; 1696 end 1697 if last_i < first_i || last_j < first_j 1698 errormsg= 'series/Run_Callback:last field index must be larger or equal to the first one'; 1699 return 1700 end 1701 %incr_i must be defined, =1 by default, if NbSlice is active 1702 if isempty(incr_i)&& ~isempty(Param.IndexRange.NbSlice) 1703 incr_i=1; 1704 set(handles.num_incr_i,'String','1') 1705 end 1706 % case of no increment i defined: processing is done on the available files found in i1_series 1707 if isempty(incr_i) 1708 if isempty(incr_j) 1709 [ref_j,ref_i]=find(squeeze(SeriesData.i1_series{1}(1,:,:))); 1710 ref_j=ref_j(ref_j>=first_j & ref_j<=last_j); 1711 ref_i=ref_i(ref_i>=first_i & ref_i<=last_i); 1712 ref_j=ref_j-1; 1713 ref_i=ref_i-1; 1610 1714 else 1611 comment=', will complement existing result files (no overwriting)'; 1612 end 1613 answer=msgbox_uvmat('INPUT_Y-N-Cancel',['use existing ouput directory: ' fullfile(Param.InputTable{1,1},SubDirOutNew) comment]); 1614 if strcmp(answer,'Cancel') 1715 ref_j=first_j:incr_j:last_j; 1716 [tild,ref_i]=find(squeeze(SeriesData.i1_series{1}(1,:,:))); 1717 ref_i=ref_i-1; 1718 ref_i=ref_i(ref_i>=first_i & ref_i<=last_i); 1719 end 1720 % increment i is defined: processing is done on first_i:incr_i:last_i; 1721 else 1722 ref_i=first_i:incr_i:last_i; 1723 if isempty(incr_j)% automatic finding of the existing j indices 1724 [ref_j,tild]=find(squeeze(SeriesData.i1_series{1}(1,:,:))); 1725 ref_j=ref_j-1; 1726 ref_j=ref_j(ref_j>=first_j & ref_j<=last_j); 1727 else 1728 ref_j=first_j:incr_j:last_j; 1729 end 1730 end 1731 CPUTime=1; % job time estimated at 1 min per iteration (on index i and j) by default 1732 if isfield(Param.Action, 'CPUTime') && ~isempty(Param.Action.CPUTime) 1733 CPUTime=Param.Action.CPUTime; % Note: CpUTime for one iteration ref_i has to be multiplied by the number of j indices nbfield_j 1734 end 1735 nbfield_j=numel(ref_j); % number of j indices 1736 BlockLength=numel(ref_i); % by default, job involves the full set of i field indices 1737 NbProcess=1; 1738 switch RunMode 1739 case 'cluster' 1740 JobNumberMax=SeriesData.SeriesParam.ClusterParam.JobNumberMax; 1741 JobCPUTimeAdvised=SeriesData.SeriesParam.ClusterParam.JobCPUTimeAdvised; 1742 if isempty(Param.IndexRange.NbSlice)% if NbSlice is not defined 1743 BlockLength= ceil(JobCPUTimeAdvised/(CPUTime*nbfield_j)); % iterations are grouped in sets with length BlockLength such that the typical CPU time of a job is JobCPUTimeAdvised. 1744 BlockLength=max(BlockLength,ceil(numel(ref_i)*NbExp/JobNumberMax)); % possibly increase the BlockLength to have less than MaxJobNumber jobs 1745 NbProcess=ceil(numel(ref_i)/BlockLength) ; % nbre of processes sent to oar 1746 else 1747 NbProcess=Param.IndexRange.NbSlice; % the parameter NbSlice sets the nbre of run processes 1748 end 1749 NbCore=min(NbCore,NbProcess); % reduces the number of cores if it exceeds the number of processes 1750 otherwise 1751 if ~isempty(Param.IndexRange.NbSlice) 1752 NbProcess=Param.IndexRange.NbSlice; % the parameter NbSlice sets the nbre of run processes 1753 end 1754 end 1755 1756 %% record nbre of output files and starting time for computation for status 1757 StatusData=get(handles.status,'UserData'); 1758 if isfield(StatusData,'OutputFileMode') 1759 switch StatusData.OutputFileMode 1760 case 'NbInput' 1761 StatusData.NbOutputFile=numel(ref_i)*nbfield_j; 1762 case 'NbInput_i' 1763 StatusData.NbOutputFile=numel(ref_i); 1764 case 'NbSlice' 1765 StatusData.NbOutputFile=str2num(get(handles.num_NbSlice,'String')); 1766 end 1767 end 1768 StatusData.TimeStart=now; 1769 set(handles.status,'UserData',StatusData) 1770 1771 %% case of a function in Python 1772 if strcmp(ActionExt, '.py (in dev.)') 1773 fprintf([ 1774 '\n' ... 1775 '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n' ... 1776 'The option .py is used. It is still in development.\n' ... 1777 'To try it, first install pyper and the most recent version of fluidimage\n' ... 1778 '(see https://bitbucket.org/fluiddyn/fluidimage).\n' ... 1779 'Warning: there is no direct correspondance between UVMAT and fluidimage parameters\n' ... 1780 '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n']) 1781 RunMode = 'python'; 1782 end 1783 1784 1785 %% direct processing on the current Matlab session or creation of command files 1786 filexml=cell(1,NbProcess); % initialisation of the names of the files containing the processing parameters 1787 extxml=cell(1,NbProcess); % initialisation of the set of labels used for the files documenting each process 1788 for iprocess=1:NbProcess 1789 extxml{iprocess}='.xml'; 1790 end 1791 for iprocess=1:NbProcess 1792 if ~strcmp(get(handles.RUN,'BusyAction'),'queue')% allow for STOP action 1793 disp('program stopped by user') 1615 1794 return 1616 elseif strcmp(answer,'Yes') 1617 detect=0; 1618 check_create=0; 1619 else 1620 r=regexp(SubDirOutNew,'(?<root>.*\D)(?<num1>\d+)$','names'); % detect whether name ends by a number 1621 if isempty(r) 1622 r(1).root=[SubDirOutNew '_']; 1623 r(1).num1='0'; 1624 end 1625 SubDirOutNew=[r(1).root num2str(str2num(r(1).num1)+1)]; % increment the index by 1 or put 1 1626 detect=exist(fullfile(Param.InputTable{1,1},SubDirOutNew),'dir'); % test if the dir already exists 1627 check_create=1; 1628 end 1629 end 1630 Param.OutputDirExt=regexprep(SubDirOutNew,Param.OutputSubDir,''); 1631 Param.OutputRootFile=Param.InputTable{1,3}; % the first sorted RootFile taken for output 1632 set(handles.OutputDirExt,'String',Param.OutputDirExt) 1633 OutputDir=fullfile(Param.InputTable{1,1},[Param.OutputSubDir Param.OutputDirExt]); % full name (with path) of output directory 1634 if check_create % create output directory if it does not exist 1635 [tild,msg1]=mkdir(OutputDir); 1636 if ~strcmp(msg1,'') 1637 errormsg=['cannot create ' OutputDir ': ' msg1]; % error message for directory creation 1638 return 1795 end 1796 if isempty(Param.IndexRange.NbSlice) 1797 Param.IndexRange.first_i=first_i+(iprocess-1)*BlockLength*incr_i; 1798 if Param.IndexRange.first_i>last_i 1799 NbProcess=iprocess-1; % leave the loop, we are at the end of the calculation 1800 break 1801 end 1802 Param.IndexRange.last_i=min(last_i,first_i+(iprocess)*BlockLength*incr_i-1); 1803 else %multislices (then incr_i is not empty) 1804 Param.IndexRange.first_i= first_i+iprocess-1; 1805 Param.IndexRange.incr_i=incr_i*Param.IndexRange.NbSlice; 1806 end 1807 for ilist=1:size(Param.InputTable,1) 1808 Param.InputTable{ilist,1}=regexprep(Param.InputTable{ilist,1},'\','/'); % correct path name for PCWIN system 1809 end 1810 1811 if isfield(Param,'OutputSubDir') 1812 t=struct2xml(Param); 1813 t=set(t,1,'name','Series'); 1814 extxml{iprocess}=fullfile_uvmat('','',Param.InputTable{1,3},'.xml',OutputNomType,... 1815 Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j); 1816 filexml{iprocess}=fullfile(OutputDir,'0_XML',extxml{iprocess}); 1817 try 1818 save(t, filexml{iprocess}); % save the xml file containing the processing parameters 1819 catch ME 1820 if ~strcmp (RunMode,'local') 1821 errormsg=['error writting ' filexml{iprocess} ': ' ME.message]; 1822 return 1823 end 1824 end 1825 end 1826 if strcmp (RunMode,'local') 1827 switch ActionExt 1828 case '.m' 1829 h_fun(Param); % direct launching 1830 1831 case '.sh' 1832 switch computer 1833 case {'PCWIN','PCWIN64'} %Windows system 1834 filexml=regexprep(filexml,'\\','\\\\'); % add '\' so that '\' are left as characters 1835 system([ActionFullName ' ' RunTime ' ' filexml{iprocess}]); % TODO: adapt to DOS system 1836 case {'GLNX86','GLNXA64','MACI64'}%Linux system 1837 system([ActionFullName ' ' RunTime ' ' filexml{iprocess}]); 1838 end 1839 end 1639 1840 end 1640 1841 end 1641 1842 1642 elseif isfield(Param,'ActionInput')&&isfield(Param.ActionInput,'LogPath')% custom definition of the output dir 1643 OutputDir=Param.ActionInput.LogPath; 1644 end 1645 DirXml=fullfile(OutputDir,'0_XML'); 1646 if ~exist(DirXml,'dir') 1647 [~,msg1]=mkdir(DirXml); 1648 if ~strcmp(msg1,'') 1649 errormsg=['cannot create ' DirXml ': ' msg1]; % error message for directory creation 1650 return 1651 end 1652 end 1653 OutputNomType=nomtype2pair(Param.InputTable{1,4}); % nomenclature for output files 1654 1655 %% get the set of reference input field indices 1656 first_i=1; % first i index to process 1657 last_i=1; % last i index to process 1658 incr_i=1; % increment step in i index 1659 first_j=1; % first j index to process 1660 last_j=1; % last j index to process 1661 incr_j=1; % increment step in j index 1662 if isfield(Param.IndexRange,'first_i') 1663 first_i=Param.IndexRange.first_i; 1664 incr_i=Param.IndexRange.incr_i; 1665 last_i=Param.IndexRange.last_i; 1666 end 1667 if isfield(Param.IndexRange,'incr_j') 1668 first_j=Param.IndexRange.first_j; 1669 last_j=Param.IndexRange.last_j; 1670 incr_j=Param.IndexRange.incr_j; 1671 end 1672 if last_i < first_i || last_j < first_j 1673 errormsg= 'series/Run_Callback:last field index must be larger or equal to the first one'; 1674 return 1675 end 1676 %incr_i must be defined, =1 by default, if NbSlice is active 1677 if isempty(incr_i)&& ~isempty(Param.IndexRange.NbSlice) 1678 incr_i=1; 1679 set(handles.num_incr_i,'String','1') 1680 end 1681 % case of no increment i defined: processing is done on the available files found in i1_series 1682 if isempty(incr_i) 1683 if isempty(incr_j) 1684 [ref_j,ref_i]=find(squeeze(SeriesData.i1_series{1}(1,:,:))); 1685 ref_j=ref_j(ref_j>=first_j & ref_j<=last_j); 1686 ref_i=ref_i(ref_i>=first_i & ref_i<=last_i); 1687 ref_j=ref_j-1; 1688 ref_i=ref_i-1; 1689 else 1690 ref_j=first_j:incr_j:last_j; 1691 [tild,ref_i]=find(squeeze(SeriesData.i1_series{1}(1,:,:))); 1692 ref_i=ref_i-1; 1693 ref_i=ref_i(ref_i>=first_i & ref_i<=last_i); 1694 end 1695 % increment i is defined: processing is done on first_i:incr_i:last_i; 1696 else 1697 ref_i=first_i:incr_i:last_i; 1698 if isempty(incr_j)% automatic finding of the existing j indices 1699 [ref_j,tild]=find(squeeze(SeriesData.i1_series{1}(1,:,:))); 1700 ref_j=ref_j-1; 1701 ref_j=ref_j(ref_j>=first_j & ref_j<=last_j); 1702 else 1703 ref_j=first_j:incr_j:last_j; 1704 end 1705 end 1706 CPUTime=1; % job time estimated at 1 min per iteration (on index i and j) by default 1707 if isfield(Param.Action, 'CPUTime') && ~isempty(Param.Action.CPUTime) 1708 CPUTime=Param.Action.CPUTime; % Note: CpUTime for one iteration ref_i has to be multiplied by the number of j indices nbfield_j 1709 end 1710 nbfield_j=numel(ref_j); % number of j indices 1711 BlockLength=numel(ref_i); % by default, job involves the full set of i field indices 1712 NbProcess=1; 1713 switch RunMode 1714 case 'cluster' 1715 JobNumberMax=SeriesData.SeriesParam.ClusterParam.JobNumberMax; 1716 JobCPUTimeAdvised=SeriesData.SeriesParam.ClusterParam.JobCPUTimeAdvised; 1717 if isempty(Param.IndexRange.NbSlice)% if NbSlice is not defined 1718 BlockLength= ceil(JobCPUTimeAdvised/(CPUTime*nbfield_j)); % iterations are grouped in sets with length BlockLength such that the typical CPU time of a job is MinJobNumber. 1719 BlockLength=max(BlockLength,ceil(numel(ref_i)/JobNumberMax)); % possibly increase the BlockLength to have less than MaxJobNumber jobs 1720 NbProcess=ceil(numel(ref_i)/BlockLength) ; % nbre of processes sent to oar 1721 else 1722 NbProcess=Param.IndexRange.NbSlice; % the parameter NbSlice sets the nbre of run processes 1723 end 1724 NbCore=min(NbCore,NbProcess); % reduces the number of cores if it exceeds the number of processes 1725 otherwise 1726 if ~isempty(Param.IndexRange.NbSlice) 1727 NbProcess=Param.IndexRange.NbSlice; % the parameter NbSlice sets the nbre of run processes 1728 end 1729 end 1730 1731 %% record nbre of output files and starting time for computation for status 1732 StatusData=get(handles.status,'UserData'); 1733 if isfield(StatusData,'OutputFileMode') 1734 switch StatusData.OutputFileMode 1735 case 'NbInput' 1736 StatusData.NbOutputFile=numel(ref_i)*nbfield_j; 1737 case 'NbInput_i' 1738 StatusData.NbOutputFile=numel(ref_i); 1739 case 'NbSlice' 1740 StatusData.NbOutputFile=str2num(get(handles.num_NbSlice,'String')); 1741 end 1742 end 1743 StatusData.TimeStart=now; 1744 set(handles.status,'UserData',StatusData) 1745 1746 %% case of a function in Python 1747 if strcmp(ActionExt, '.py (in dev.)') 1748 fprintf([ 1749 '\n' ... 1750 '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n' ... 1751 'The option .py is used. It is still in development.\n' ... 1752 'To try it, first install pyper and the most recent version of fluidimage\n' ... 1753 '(see https://bitbucket.org/fluiddyn/fluidimage).\n' ... 1754 'Warning: there is no direct correspondance between UVMAT and fluidimage parameters\n' ... 1755 '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n']) 1756 RunMode = 'python'; 1757 end 1758 1759 1760 %% direct processing on the current Matlab session or creation of command files 1761 filexml=cell(1,NbProcess); % initialisation of the names of the files containing the processing parameters 1762 extxml=cell(1,NbProcess); % initialisation of the set of labels used for the files documenting each process 1763 for iprocess=1:NbProcess 1764 extxml{iprocess}='.xml'; 1765 end 1766 for iprocess=1:NbProcess 1767 if ~strcmp(get(handles.RUN,'BusyAction'),'queue')% allow for STOP action 1768 disp('program stopped by user') 1769 return 1770 end 1771 if isempty(Param.IndexRange.NbSlice) 1772 Param.IndexRange.first_i=first_i+(iprocess-1)*BlockLength*incr_i; 1773 if Param.IndexRange.first_i>last_i 1774 NbProcess=iprocess-1; % leave the loop, we are at the end of the calculation 1775 break 1776 end 1777 Param.IndexRange.last_i=min(last_i,first_i+(iprocess)*BlockLength*incr_i-1); 1778 else %multislices (then incr_i is not empty) 1779 Param.IndexRange.first_i= first_i+iprocess-1; 1780 Param.IndexRange.incr_i=incr_i*Param.IndexRange.NbSlice; 1781 end 1782 for ilist=1:size(Param.InputTable,1) 1783 Param.InputTable{ilist,1}=regexprep(Param.InputTable{ilist,1},'\','/'); % correct path name for PCWIN system 1843 if ~strcmp (RunMode,'local') && ~strcmp(RunMode,'python') 1844 %% processing on a different session of the same computer (background) or cluster, create executable files 1845 batch_file_list=cell(NbProcess,1); % initiate the list of executable files 1846 DirExe=fullfile(OutputDir,'0_EXE'); % directory name for executable files 1847 switch computer 1848 case {'PCWIN','PCWIN64'} %Windows system 1849 ExeExt='.bat'; 1850 case {'GLNX86','GLNXA64','MACI64'}%Linux system 1851 ExeExt='.sh'; 1852 end 1853 %create subdirectory for executable files 1854 if ~exist(DirExe,'dir') 1855 [tild,msg1]=mkdir(DirExe); 1856 if ~strcmp(msg1,'') 1857 errormsg=['cannot create ' DirExe ': ' msg1]; % error message for directory creation 1858 return 1859 end 1860 end 1861 %create subdirectory for log files 1862 DirLog=fullfile(OutputDir,'0_LOG'); 1863 if ~exist(DirLog,'dir') 1864 [tild,msg1]=mkdir(DirLog); 1865 if ~strcmp(msg1,'') 1866 errormsg=['cannot create ' DirLog ': ' msg1]; % error message for directory creation 1867 return 1868 end 1869 end 1870 1871 %create the executable file 1872 file_exe_global=fullfile_uvmat('','',Param.InputTable{1,3},ExeExt,OutputNomType,... 1873 first_i,last_i,first_j,last_j); 1874 file_exe_global=fullfile(OutputDir,'0_EXE',file_exe_global); 1875 filelog_global=fullfile_uvmat('','',Param.InputTable{1,3},'.log',OutputNomType,... 1876 first_i,last_i,first_j,last_j); 1877 filelog_global=fullfile(OutputDir,'0_LOG',filelog_global); 1878 1879 for iprocess=1:NbProcess 1880 %create the executable file 1881 batch_file_list{iprocess}=fullfile(OutputDir,'0_EXE',regexprep(extxml{iprocess},'.xml$',ExeExt)); 1882 1883 % set the log file name 1884 filelog{iprocess}=fullfile(OutputDir,'0_LOG',regexprep(extxml{iprocess},'.xml$','.log')); 1885 end 1784 1886 end 1785 1887 1786 if isfield(Param,'OutputSubDir') 1787 t=struct2xml(Param); 1788 t=set(t,1,'name','Series'); 1789 extxml{iprocess}=fullfile_uvmat('','',Param.InputTable{1,3},'.xml',OutputNomType,... 1790 Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j); 1791 filexml{iprocess}=fullfile(OutputDir,'0_XML',extxml{iprocess}); 1792 try 1793 save(t, filexml{iprocess}); % save the xml file containing the processing parameters 1794 catch ME 1795 if ~strcmp (RunMode,'local') 1796 errormsg=['error writting ' filexml{iprocess} ': ' ME.message]; 1888 %% launch the executable files for background or cluster processing 1889 1890 switch RunMode 1891 1892 case 'background' 1893 [fid,message]=fopen(file_exe_global,'w'); 1894 if isequal(fid,-1) 1895 errormsg=['creation of ' file_exe_global ':' message]; 1797 1896 return 1798 1897 end 1799 end 1800 end 1801 if strcmp (RunMode,'local') 1802 switch ActionExt 1803 case '.m' 1804 h_fun(Param); % direct launching 1805 1806 case '.sh' 1807 switch computer 1808 case {'PCWIN','PCWIN64'} %Windows system 1809 filexml=regexprep(filexml,'\\','\\\\'); % add '\' so that '\' are left as characters 1810 system([ActionFullName ' ' RunTime ' ' filexml{iprocess}]); % TODO: adapt to DOS system 1811 case {'GLNX86','GLNXA64','MACI64'}%Linux system 1812 system([ActionFullName ' ' RunTime ' ' filexml{iprocess}]); 1813 end 1814 end 1815 end 1816 end 1817 1818 if ~strcmp (RunMode,'local') && ~strcmp(RunMode,'python') 1819 %% processing on a different session of the same computer (background) or cluster, create executable files 1820 batch_file_list=cell(NbProcess,1); % initiate the list of executable files 1821 DirExe=fullfile(OutputDir,'0_EXE'); % directory name for executable files 1822 switch computer 1823 case {'PCWIN','PCWIN64'} %Windows system 1824 ExeExt='.bat'; 1825 case {'GLNX86','GLNXA64','MACI64'}%Linux system 1826 ExeExt='.sh'; 1827 end 1828 %create subdirectory for executable files 1829 if ~exist(DirExe,'dir') 1830 [tild,msg1]=mkdir(DirExe); 1831 if ~strcmp(msg1,'') 1832 errormsg=['cannot create ' DirExe ': ' msg1]; % error message for directory creation 1833 return 1834 end 1835 end 1836 %create subdirectory for log files 1837 DirLog=fullfile(OutputDir,'0_LOG'); 1838 if ~exist(DirLog,'dir') 1839 [tild,msg1]=mkdir(DirLog); 1840 if ~strcmp(msg1,'') 1841 errormsg=['cannot create ' DirLog ': ' msg1]; % error message for directory creation 1842 return 1843 end 1844 end 1845 1846 %create the executable file 1847 file_exe_global=fullfile_uvmat('','',Param.InputTable{1,3},ExeExt,OutputNomType,... 1848 first_i,last_i,first_j,last_j); 1849 file_exe_global=fullfile(OutputDir,'0_EXE',file_exe_global); 1850 filelog_global=fullfile_uvmat('','',Param.InputTable{1,3},'.log',OutputNomType,... 1851 first_i,last_i,first_j,last_j); 1852 filelog_global=fullfile(OutputDir,'0_LOG',filelog_global); 1853 1854 for iprocess=1:NbProcess 1855 %create the executable file 1856 batch_file_list{iprocess}=fullfile(OutputDir,'0_EXE',regexprep(extxml{iprocess},'.xml$',ExeExt)); 1857 1858 % set the log file name 1859 filelog{iprocess}=fullfile(OutputDir,'0_LOG',regexprep(extxml{iprocess},'.xml$','.log')); 1860 end 1861 end 1862 1863 %% launch the executable files for background or cluster processing 1864 1865 switch RunMode 1866 1867 case 'background' 1868 [fid,message]=fopen(file_exe_global,'w'); 1869 if isequal(fid,-1) 1870 errormsg=['creation of ' file_exe_global ':' message]; 1871 return 1872 end 1873 switch ActionExt 1874 case '.m'% Matlab function 1875 switch computer 1876 case {'GLNX86','GLNXA64','MACI64'} 1877 matlab_ver = ver('MATLAB'); 1878 matlab_version = matlab_ver.Version; 1879 cmd=[... 1880 '#!/bin/bash\n'... 1881 'source /etc/profile\n'... 1882 'module load matlab/' matlab_version '\n'...% CHOICE OF MATLAB VERSION 1883 'time_start=$(date +%%s)\n'... 1884 'matlab -nodisplay -nosplash -nojvm -logfile ''' filelog_global ''' <<END_MATLAB\n'... 1885 'addpath(''' path_series ''');\n'... 1886 'addpath(''' Param.Action.ActionPath ''');\n']; 1887 for iprocess=1:NbProcess 1888 cmd=[cmd '' Param.Action.ActionName '(''' filexml{iprocess} ''');\n']; 1889 end 1890 cmd=[cmd 'exit\n' 'END_MATLAB\n'... 1891 'time_end=$(date +%%s)\n'... 1892 'echo "global time = " $(($time_end - $time_start)) >> ''' filelog_global '''\n']; 1893 fprintf(fid,cmd); % fill the executable file with the char string cmd 1894 fclose(fid); % close the executable filefilelog_global 1895 system(['chmod +x ' file_exe_global]); % set the file to executable 1896 case {'PCWIN','PCWIN64'} 1897 cmd=['matlab -automation -logfile ' regexprep(filelog{iprocess},'\\','\\\\')... 1898 ' -r "addpath(''' regexprep(path_series,'\\','\\\\') ''');'... 1899 'addpath(''' regexprep(Param.Action.ActionPath,'\\','\\\\') ''');']; 1900 for iprocess=1:NbProcess 1901 cmd=[cmd '' Param.Action.ActionName '( ''' regexprep(filexml{iprocess},'\\','\\\\') ''');'] 1902 end 1903 cmd=[cmd ';exit"']; 1904 fprintf(fid,cmd); % fill the executable file with the char string cmd 1905 fclose(fid); % close the executable file 1906 end 1907 system([file_exe_global ' &'])% directly execute the command file 1908 case '.sh' % compiled Matlab function 1909 for iprocess=1:NbProcess 1898 switch ActionExt 1899 case '.m'% Matlab function 1910 1900 switch computer 1911 1901 case {'GLNX86','GLNXA64','MACI64'} 1912 [fid,message]=fopen(batch_file_list{iprocess},'w'); % create the executable file 1913 if isequal(fid,-1) 1914 errormsg=['creation of .bat file: ' message]; 1915 return 1902 matlab_ver = ver('MATLAB'); 1903 matlab_version = matlab_ver.Version; 1904 cmd=[... 1905 '#!/bin/bash\n'... 1906 'source /etc/profile\n'... 1907 'module load matlab/' matlab_version '\n'...% CHOICE OF MATLAB VERSION 1908 'time_start=$(date +%%s)\n'... 1909 'matlab -nodisplay -nosplash -nojvm -logfile ''' filelog_global ''' <<END_MATLAB\n'... 1910 'addpath(''' path_series ''');\n'... 1911 'addpath(''' Param.Action.ActionPath ''');\n']; 1912 for iprocess=1:NbProcess 1913 cmd=[cmd '' Param.Action.ActionName '(''' filexml{iprocess} ''');\n']; 1916 1914 end 1917 cmd=['#!/bin/bash \n '... 1918 '#$ -cwd \n '... 1919 'hostname && date \n '... 1920 'umask 002 \n'... 1921 ActionFullName ' ' RunTime ' ' filexml{iprocess}]; % allow writting access to created files for user group 1915 cmd=[cmd 'exit\n' 'END_MATLAB\n'... 1916 'time_end=$(date +%%s)\n'... 1917 'echo "global time = " $(($time_end - $time_start)) >> ''' filelog_global '''\n']; 1918 fprintf(fid,cmd); % fill the executable file with the char string cmd 1919 fclose(fid); % close the executable filefilelog_global 1920 system(['chmod +x ' file_exe_global]); % set the file to executable 1921 case {'PCWIN','PCWIN64'} 1922 cmd=['matlab -automation -logfile ' regexprep(filelog{iprocess},'\\','\\\\')... 1923 ' -r "addpath(''' regexprep(path_series,'\\','\\\\') ''');'... 1924 'addpath(''' regexprep(Param.Action.ActionPath,'\\','\\\\') ''');']; 1925 for iprocess=1:NbProcess 1926 cmd=[cmd '' Param.Action.ActionName '( ''' regexprep(filexml{iprocess},'\\','\\\\') ''');'] 1927 end 1928 cmd=[cmd ';exit"']; 1922 1929 fprintf(fid,cmd); % fill the executable file with the char string cmd 1923 1930 fclose(fid); % close the executable file 1924 system(['chmod +x ' batch_file_list{iprocess}]); % set the file to executable1925 system([batch_file_list{iprocess} ' &'])% directly execute the command file1926 case {'PCWIN','PCWIN64'}1927 msgbox_uvmat('ERROR','option for compiled Matlab functions not implemented for Windows system')1928 return1929 1931 end 1932 system([file_exe_global ' &'])% directly execute the command file 1933 case '.sh' % compiled Matlab function 1934 for iprocess=1:NbProcess 1935 switch computer 1936 case {'GLNX86','GLNXA64','MACI64'} 1937 [fid,message]=fopen(batch_file_list{iprocess},'w'); % create the executable file 1938 if isequal(fid,-1) 1939 errormsg=['creation of .bat file: ' message]; 1940 return 1941 end 1942 cmd=['#!/bin/bash \n '... 1943 '#$ -cwd \n '... 1944 'hostname && date \n '... 1945 'umask 002 \n'... 1946 ActionFullName ' ' RunTime ' ' filexml{iprocess}]; % allow writting access to created files for user group 1947 fprintf(fid,cmd); % fill the executable file with the char string cmd 1948 fclose(fid); % close the executable file 1949 system(['chmod +x ' batch_file_list{iprocess}]); % set the file to executable 1950 system([batch_file_list{iprocess} ' &'])% directly execute the command file 1951 case {'PCWIN','PCWIN64'} 1952 msgbox_uvmat('ERROR','option for compiled Matlab functions not implemented for Windows system') 1953 return 1954 end 1955 end 1956 msgbox_uvmat('CONFIRMATION',[ActionFullName ' launched in background for ' ExpName ': press STATUS to see results']) 1957 end 1958 1959 case 'cluster' % option 'oar-parexec' used 1960 %create subdirectory for oar commands 1961 for iprocess=1:NbProcess 1962 [fid,message]=fopen(batch_file_list{iprocess},'w'); % create the executable file 1963 if isequal(fid,-1) 1964 errormsg=['creation of .bat file: ' message]; 1965 return 1930 1966 end 1931 msgbox_uvmat('CONFIRMATION',[ActionFullName ' launched in background: press STATUS to see results']) 1932 end 1933 1934 case 'cluster' % option 'oar-parexec' used 1935 %create subdirectory for oar commands 1936 for iprocess=1:NbProcess 1937 [fid,message]=fopen(batch_file_list{iprocess},'w'); % create the executable file 1938 if isequal(fid,-1) 1939 errormsg=['creation of .bat file: ' message]; 1940 return 1941 end 1942 if strcmp(ActionExt,'.sh') 1943 cmd=['#!/bin/bash \n '... 1944 '#$ -cwd \n '... 1945 'hostname && date \n '... 1946 'umask 002 \n'... 1947 ActionFullName ' ' RunTime ' ' filexml{iprocess}]; % allow writting access to created files for user group 1967 if strcmp(ActionExt,'.sh') 1968 cmd=['#!/bin/bash \n '... 1969 '#$ -cwd \n '... 1970 'hostname && date \n '... 1971 'umask 002 \n'... 1972 ActionFullName ' ' RunTime ' ' filexml{iprocess}]; % allow writting access to created files for user group 1973 else 1974 matlab_ver = ver('MATLAB'); 1975 matlab_version = matlab_ver.Version; 1976 cmd=[... 1977 '#!/bin/bash\n'... 1978 'source /etc/profile\n'... 1979 'module load matlab/' matlab_version '\n'...% CHOICE OF MATLAB VERSION 1980 'matlab -nodisplay -nosplash -nojvm -singleCompThread -logfile ''' filelog{iprocess} ''' <<END_MATLAB\n'... 1981 'addpath(''' path_series ''');\n'... 1982 'addpath(''' Param.Action.ActionPath ''');\n'... 1983 '' Param.Action.ActionName '(''' filexml{iprocess} ''');\n'... 1984 'exit\n'... 1985 'END_MATLAB\n']; 1986 end 1987 fprintf(fid,cmd); % fill the executable file with the char string cmd 1988 fclose(fid); % close the executable file 1989 system(['chmod +x ' batch_file_list{iprocess}]); % set the file to executable 1990 end 1991 DIR_CLUSTER=fullfile(OutputDir,'0_CLUSTER'); 1992 if exist(DIR_CLUSTER,'dir')% delete the content of the dir 0_LOG to allow new input 1993 curdir=pwd; 1994 cd(DIR_CLUSTER) 1995 delete('*') 1996 cd(curdir) 1948 1997 else 1949 matlab_ver = ver('MATLAB'); 1950 matlab_version = matlab_ver.Version; 1951 cmd=[... 1952 '#!/bin/bash\n'... 1953 'source /etc/profile\n'... 1954 'module load matlab/' matlab_version '\n'...% CHOICE OF MATLAB VERSION 1955 'matlab -nodisplay -nosplash -nojvm -singleCompThread -logfile ''' filelog{iprocess} ''' <<END_MATLAB\n'... 1956 'addpath(''' path_series ''');\n'... 1957 'addpath(''' Param.Action.ActionPath ''');\n'... 1958 '' Param.Action.ActionName '(''' filexml{iprocess} ''');\n'... 1959 'exit\n'... 1960 'END_MATLAB\n']; 1961 end 1962 fprintf(fid,cmd); % fill the executable file with the char string cmd 1963 fclose(fid); % close the executable file 1964 system(['chmod +x ' batch_file_list{iprocess}]); % set the file to executable 1965 end 1966 DIR_CLUSTER=fullfile(OutputDir,'0_CLUSTER'); 1967 if exist(DIR_CLUSTER,'dir')% delete the content of the dir 0_LOG to allow new input 1968 curdir=pwd; 1969 cd(DIR_CLUSTER) 1970 delete('*') 1971 cd(curdir) 1972 else 1973 [tild,msg1]=mkdir(DIR_CLUSTER); 1974 if ~strcmp(msg1,'') 1975 errormsg=['cannot create ' DIR_CLUSTER ': ' msg1]; % error message for directory creation 1976 return 1977 end 1978 end 1979 % create file containing the list of jobs 1980 ListProcess=fullfile(DIR_CLUSTER,'job_list.txt'); % name of the file containing the list of executables 1981 fid=fopen(ListProcess,'w'); % open it for writting 1982 for iprocess=1:length(batch_file_list) 1983 fprintf(fid,[batch_file_list{iprocess} '\n']); % write list of exe files 1984 end 1985 fclose(fid); 1986 system(['chmod +x ' ListProcess]); % set the file to executable 1987 1988 CPUTimeProcess=CPUTime*BlockLength*nbfield_j; % estimated CPU time for one individual process (in minutes) 1989 LaunchCmdFcn=SeriesData.SeriesParam.ClusterParam.LaunchCmdFcn; 1990 oar_command=feval(LaunchCmdFcn,ListProcess,ActionFullName,DirLog,NbProcess, NbCore,CPUTimeProcess) 1991 [status,result]=system(oar_command)% execute system command and show the result (ID number of the launched job) on the Matlab command window 1992 filename_oarcommand=fullfile(DIR_CLUSTER,'0_cluster_command'); % keep track of the command in file '0-OAR/0_cluster_command' 1993 fid=fopen(filename_oarcommand,'w'); 1994 fprintf(fid,oar_command); % store the command 1995 fprintf(fid,result); % store the result (job ID number) 1996 fclose(fid); 1997 msgbox_uvmat('CONFIRMATION',[ActionFullName ' launched as ' num2str(NbProcess) ' processes in cluster: press STATUS to see results']) 1998 1999 % case 'cluster_pbs' % for LMFA Kepler machine: trqnsferred to fct 2000 2001 % %create subdirectory for pbs command and log files 2002 % DirPBS=fullfile(OutputDir,'0_PBS'); % todo : common name OAR/PBS 2003 % if exist(DirPBS,'dir')% delete the content of the dir 0_LOG to allow new input 2004 % curdir=pwd; 2005 % cd(DirPBS) 2006 % delete('*') 2007 % cd(curdir) 2008 % else 2009 % [tild,msg1]=mkdir(DirPBS); 2010 % if ~strcmp(msg1,'') 2011 % errormsg=['cannot create ' DirPBS ': ' msg1]; % error message for directory creation 2012 % return 2013 % end 2014 % end 2015 % max_walltime=3600*20; % 20h max total calculation (cannot exceed 24 h) 2016 % walltime_onejob=1800; % seconds, max estimated time for asingle file index value 2017 % ListProcess=fullfile(DirPBS,'job_list.txt'); % create name of the global executable file 2018 % fid=fopen(ListProcess,'w'); 2019 % for iprocess=1:length(batch_file_list) 2020 % fprintf(fid,[batch_file_list{iprocess} '\n']); % list of exe files 2021 % end 2022 % fclose(fid); 2023 % system(['chmod +x ' ListProcess]); % set the file to executable 2024 % pbs_command=['qsub -n CIVX '... 2025 % '-t idempotent --checkpoint ' num2str(walltime_onejob+60) ' '... 2026 % '-l /core=' num2str(NbCore) ','... 2027 % 'walltime=' datestr(min(1.05*walltime_onejob/86400*max(NbProcess*BlockLength*nbfield_j,NbCore)/NbCore,max_walltime/86400),13) ' '... 2028 % '-E ' regexprep(ListProcess,'\.txt\>','.stderr') ' '... 2029 % '-O ' regexprep(ListProcess,'\.txt\>','.log') ' '... 2030 % extra_qstat ' '... 2031 % '"oar-parexec -s -f ' ListProcess ' '... 2032 % '-l ' ListProcess '.log"']; 2033 % filename_oarcommand=fullfile(DirPBS,'pbs_command'); 2034 % fid=fopen(filename_oarcommand,'w'); 2035 % fprintf(fid,pbs_command); 2036 % fclose(fid); 2037 % fprintf(pbs_command); % display in command line 2038 % %system(pbs_command); 2039 % msgbox_uvmat('CONFIRMATION',[ActionFullName ' command ready to be launched in cluster']) 2040 2041 case 'cluster_sge' % for PSMN % TODO: use the standard 'cluster' config with an external fct 2042 % Au PSMN, on ne cr??e pas 1 job avec plusieurs c??urs, mais N jobs de 1 c??urs 2043 % o?? N < 1000. 2044 %create subdirectory for pbs command and log files 2045 2046 DirSGE=fullfile(OutputDir,'0_SGE'); 2047 if exist(DirSGE,'dir')% delete the content of the dir 0_LOG to allow new input 2048 curdir=pwd; 2049 cd(DirSGE) 2050 delete('*') 2051 cd(curdir) 2052 else 2053 [tild,msg1]=mkdir(DirSGE); 2054 if ~strcmp(msg1,'') 2055 errormsg=['cannot create ' DirSGE ': ' msg1]; % error message for directory creation 2056 return 2057 end 2058 end 2059 maxImgsPerJob = ceil(length(batch_file_list)/NbCore); 2060 disp(['Max number of jobs: ' num2str(NbCore)]) 2061 disp(['Images per job: ' num2str(maxImgsPerJob)]) 2062 2063 iprocess = 1; 2064 imgsInJob = []; 2065 currJobIndex = 1; 2066 done = 0; 2067 while(~done) 2068 if(iprocess <= length(batch_file_list)) 2069 imgsInJob = [imgsInJob, iprocess]; 2070 end 2071 if((numel(imgsInJob) >= maxImgsPerJob) || (iprocess == length(batch_file_list))) 2072 cmd=['#!/bin/sh \n'... 2073 '#$ -cwd \n'... 2074 'hostname && date\n'] 2075 for ii=1:numel(imgsInJob) 2076 cmd=[cmd ActionFullName ' /softs/matlab ' filexml{imgsInJob(ii)} '\n']; 1998 [tild,msg1]=mkdir(DIR_CLUSTER); 1999 if ~strcmp(msg1,'') 2000 errormsg=['cannot create ' DIR_CLUSTER ': ' msg1]; % error message for directory creation 2001 return 2077 2002 end 2078 [fid, message] = fopen([DirSGE '/job' num2str(currJobIndex) '.sh'], 'w'); 2079 fprintf(fid, cmd); 2080 fclose(fid); 2081 system(['chmod +x ' DirSGE '/job' num2str(currJobIndex) '.sh']) 2082 sge_command=['qsub -N civ_' num2str(currJobIndex) ' '... 2083 '-q ' qstat_Queue ' '... 2084 '-e ' fullfile([DirSGE '/job' num2str(currJobIndex) '.out']) ' '... 2085 '-o ' fullfile([DirSGE '/job' num2str(currJobIndex) '.out']) ' '... 2086 fullfile([DirSGE '/job' num2str(currJobIndex) '.sh'])]; 2087 fprintf(sge_command); % display in command line 2088 [status, result] = system(sge_command); 2089 fprintf(result); 2090 currJobIndex = currJobIndex + 1; 2091 imgsInJob = []; 2092 end 2093 if(iprocess == length(batch_file_list)) 2094 done = 1; 2095 end 2096 iprocess = iprocess + 1; 2097 end 2098 msgbox_uvmat('CONFIRMATION',[num2str(currJobIndex-1) ' jobs launched on queue ' qstat_Queue '.']) 2099 case 'python' 2100 command = ['LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | pyp "l = x.split('':''); l = [s for s in l if ''matlab'' not in s]; print('':''.join(l))") ' ... 2101 'python -m fluidimage.run_from_xml ' filexml{iprocess}]; 2102 fprintf(['command:\n' command '\n\n']) 2103 system(command, '-echo'); 2104 end 2105 if exist(OutputDir,'dir') 2106 [SUCCESS,MESSAGE,MESSAGEID] = fileattrib (OutputDir); 2107 if MESSAGE.GroupWrite~=1 2108 [success,msg] = fileattrib(OutputDir,'+w','g','s'); % allow writing access for the group of users, recursively in the folder 2109 if success==0 2110 msgbox_uvmat('WARNING',{['unable to set group write access to ' OutputDir ':']; msg}); % error message for directory creation 2111 end 2112 end 2113 end 2114 2003 end 2004 % create file containing the list of jobs 2005 ListProcess=fullfile(DIR_CLUSTER,'job_list.txt'); % name of the file containing the list of executables 2006 fid=fopen(ListProcess,'w'); % open it for writting 2007 for iprocess=1:length(batch_file_list) 2008 fprintf(fid,[batch_file_list{iprocess} '\n']); % write list of exe files 2009 end 2010 fclose(fid); 2011 system(['chmod +x ' ListProcess]); % set the file to executable 2012 2013 CPUTimeProcess=CPUTime*BlockLength*nbfield_j; % estimated CPU time for one individual process (in minutes) 2014 LaunchCmdFcn=SeriesData.SeriesParam.ClusterParam.LaunchCmdFcn; 2015 oar_command=feval(LaunchCmdFcn,ListProcess,ActionFullName,DirLog,NbProcess, NbCore,CPUTimeProcess) 2016 [status,result]=system(oar_command)% execute system command and show the result (ID number of the launched job) on the Matlab command window 2017 filename_oarcommand=fullfile(DIR_CLUSTER,'0_cluster_command'); % keep track of the command in file '0-OAR/0_cluster_command' 2018 fid=fopen(filename_oarcommand,'w'); 2019 fprintf(fid,oar_command); % store the command 2020 fprintf(fid,result); % store the result (job ID number) 2021 fclose(fid); 2022 if status==0 2023 msgbox_uvmat('CONFIRMATION',[ActionFullName ' launched for ' ExpName ' as ' num2str(NbProcess) ' processes in cluster: press STATUS to see results']) 2024 else 2025 msgbox_uvmat('ERROR',result) 2026 end 2027 % case 'cluster_pbs' % for LMFA Kepler machine: trqnsferred to fct 2028 2029 % %create subdirectory for pbs command and log files 2030 % DirPBS=fullfile(OutputDir,'0_PBS'); % todo : common name OAR/PBS 2031 % if exist(DirPBS,'dir')% delete the content of the dir 0_LOG to allow new input 2032 % curdir=pwd; 2033 % cd(DirPBS) 2034 % delete('*') 2035 % cd(curdir) 2036 % else 2037 % [tild,msg1]=mkdir(DirPBS); 2038 % if ~strcmp(msg1,'') 2039 % errormsg=['cannot create ' DirPBS ': ' msg1]; % error message for directory creation 2040 % return 2041 % end 2042 % end 2043 % max_walltime=3600*20; % 20h max total calculation (cannot exceed 24 h) 2044 % walltime_onejob=1800; % seconds, max estimated time for asingle file index value 2045 % ListProcess=fullfile(DirPBS,'job_list.txt'); % create name of the global executable file 2046 % fid=fopen(ListProcess,'w'); 2047 % for iprocess=1:length(batch_file_list) 2048 % fprintf(fid,[batch_file_list{iprocess} '\n']); % list of exe files 2049 % end 2050 % fclose(fid); 2051 % system(['chmod +x ' ListProcess]); % set the file to executable 2052 % pbs_command=['qsub -n CIVX '... 2053 % '-t idempotent --checkpoint ' num2str(walltime_onejob+60) ' '... 2054 % '-l /core=' num2str(NbCore) ','... 2055 % 'walltime=' datestr(min(1.05*walltime_onejob/86400*max(NbProcess*BlockLength*nbfield_j,NbCore)/NbCore,max_walltime/86400),13) ' '... 2056 % '-E ' regexprep(ListProcess,'\.txt\>','.stderr') ' '... 2057 % '-O ' regexprep(ListProcess,'\.txt\>','.log') ' '... 2058 % extra_qstat ' '... 2059 % '"oar-parexec -s -f ' ListProcess ' '... 2060 % '-l ' ListProcess '.log"']; 2061 % filename_oarcommand=fullfile(DirPBS,'pbs_command'); 2062 % fid=fopen(filename_oarcommand,'w'); 2063 % fprintf(fid,pbs_command); 2064 % fclose(fid); 2065 % fprintf(pbs_command); % display in command line 2066 % %system(pbs_command); 2067 % msgbox_uvmat('CONFIRMATION',[ActionFullName ' command ready to be launched in cluster']) 2068 2069 case 'cluster_sge' % for PSMN % TODO: use the standard 'cluster' config with an external fct 2070 % Au PSMN, on ne cr??e pas 1 job avec plusieurs c??urs, mais N jobs de 1 c??urs 2071 % o?? N < 1000. 2072 %create subdirectory for pbs command and log files 2073 2074 DirSGE=fullfile(OutputDir,'0_SGE'); 2075 if exist(DirSGE,'dir')% delete the content of the dir 0_LOG to allow new input 2076 curdir=pwd; 2077 cd(DirSGE) 2078 delete('*') 2079 cd(curdir) 2080 else 2081 [tild,msg1]=mkdir(DirSGE); 2082 if ~strcmp(msg1,'') 2083 errormsg=['cannot create ' DirSGE ': ' msg1]; % error message for directory creation 2084 return 2085 end 2086 end 2087 maxImgsPerJob = ceil(length(batch_file_list)/NbCore); 2088 disp(['Max number of jobs: ' num2str(NbCore)]) 2089 disp(['Images per job: ' num2str(maxImgsPerJob)]) 2090 2091 iprocess = 1; 2092 imgsInJob = []; 2093 currJobIndex = 1; 2094 done = 0; 2095 while(~done) 2096 if(iprocess <= length(batch_file_list)) 2097 imgsInJob = [imgsInJob, iprocess]; 2098 end 2099 if((numel(imgsInJob) >= maxImgsPerJob) || (iprocess == length(batch_file_list))) 2100 cmd=['#!/bin/sh \n'... 2101 '#$ -cwd \n'... 2102 'hostname && date\n'] 2103 for ii=1:numel(imgsInJob) 2104 cmd=[cmd ActionFullName ' /softs/matlab ' filexml{imgsInJob(ii)} '\n']; 2105 end 2106 [fid, message] = fopen([DirSGE '/job' num2str(currJobIndex) '.sh'], 'w'); 2107 fprintf(fid, cmd); 2108 fclose(fid); 2109 system(['chmod +x ' DirSGE '/job' num2str(currJobIndex) '.sh']) 2110 sge_command=['qsub -N civ_' num2str(currJobIndex) ' '... 2111 '-q ' qstat_Queue ' '... 2112 '-e ' fullfile([DirSGE '/job' num2str(currJobIndex) '.out']) ' '... 2113 '-o ' fullfile([DirSGE '/job' num2str(currJobIndex) '.out']) ' '... 2114 fullfile([DirSGE '/job' num2str(currJobIndex) '.sh'])]; 2115 fprintf(sge_command); % display in command line 2116 [status, result] = system(sge_command); 2117 fprintf(result); 2118 currJobIndex = currJobIndex + 1; 2119 imgsInJob = []; 2120 end 2121 if(iprocess == length(batch_file_list)) 2122 done = 1; 2123 end 2124 iprocess = iprocess + 1; 2125 end 2126 msgbox_uvmat('CONFIRMATION',[num2str(currJobIndex-1) ' jobs launched on queue ' qstat_Queue '.']) 2127 case 'python' 2128 command = ['LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | pyp "l = x.split('':''); l = [s for s in l if ''matlab'' not in s]; print('':''.join(l))") ' ... 2129 'python -m fluidimage.run_from_xml ' filexml{iprocess}]; 2130 fprintf(['command:\n' command '\n\n']) 2131 system(command, '-echo'); 2132 end 2133 if exist(OutputDir,'dir') 2134 [SUCCESS,MESSAGE,MESSAGEID] = fileattrib (OutputDir); 2135 if MESSAGE.GroupWrite~=1 2136 [success,msg] = fileattrib(OutputDir,'+w','g','s'); % allow writing access for the group of users, recursively in the folder 2137 if success==0 2138 msgbox_uvmat('WARNING',{['unable to set group write access to ' OutputDir ':']; msg}); % error message for directory creation 2139 end 2140 end 2141 end 2142 end 2115 2143 %------------------------------------------------------------------------ 2116 2144 function STOP_Callback(hObject, eventdata, handles) … … 3474 3502 end 3475 3503 3476 %function num_NbProcess_Callback(hObject, eventdata, handles)3477 3478 3504 3479 3505 function num_NbSlice_Callback(hObject, eventdata, handles) 3480 3506 NbSlice=str2num(get(handles.num_NbSlice,'String')); 3481 %set(handles.num_NbProcess,'String',num2str(NbSlice))3482 3507 3483 3508 %------------------------------------------------------------------------ … … 3520 3545 3521 3546 % --- Executes on mouse motion over figure - except title and menu. 3522 function series_WindowButtonMotionFcn(hObject, eventdata, handles)3523 set(hObject,'Pointer','arrow');3547 % function series_WindowButtonMotionFcn(hObject, eventdata, handles) 3548 % set(hObject,'Pointer','arrow'); 3524 3549 3525 3550 … … 3736 3761 3737 3762 3763 % --- Executes on button press in Replicate. 3764 function Replicate_Callback(hObject, eventdata, handles) 3765 if get(handles.Replicate,'Value') 3766 InputTable=get(handles.InputTable,'Data'); 3767 browse_data(fullfile(InputTable{1,1},InputTable{1,2})) 3768 else 3769 hh=findobj(allchild(0),'Tag','browse_data'); 3770 if ~isempty(hh) 3771 delete(hh) 3772 end 3773 end
Note: See TracChangeset
for help on using the changeset viewer.
