Changeset 917
- Timestamp:
- Jun 25, 2015, 12:33:52 PM (9 years ago)
- Location:
- trunk/src
- Files:
-
- 1 added
- 2 edited
-
series.m (modified) (8 diffs)
-
series/sub_background.m (modified) (5 diffs)
-
transform_field/ima_edge_detection.m (added)
Legend:
- Unmodified
- Added
- Removed
-
trunk/src/series.m
r916 r917 1553 1553 %% set nbre of cluster cores and processes: 1554 1554 % NbCore is the number of computer processors used 1555 % NbProcess is the number of independent processes in which the required calculation is split 1555 % NbProcess is the number of independent processes in which the required calculation is split. 1556 1556 switch RunMode 1557 1557 case {'local','background'} … … 1730 1730 NbCore=min(NbCore,NbProcess);% reduces the number of cores if it exceeds the number of processes 1731 1731 end 1732 otherwise 1733 if ~isempty(Param.IndexRange.NbSlice) 1734 NbProcess=Param.IndexRange.NbSlice;% the parameter NbSlice sets the nbre of run processes 1735 end 1732 1736 end 1733 1737 … … 1768 1772 if strcmp (RunMode,'local') 1769 1773 for iprocess=1:NbProcess 1774 if ~strcmp(get(handles.RUN,'BusyAction'),'queue')% allow for STOP action 1775 disp('program stopped by user') 1776 return 1777 end 1770 1778 if isempty(Param.IndexRange.NbSlice) 1771 1779 %Param.IndexRange.first_i=first_i+(iprocess-1)*BlockLength*incr_i; … … 1808 1816 ExeExt='.bat'; 1809 1817 case {'GLNX86','GLNXA64','MACI64'}%Linux system 1810 ExeExt='.sh';1818 ExeExt='.sh'; 1811 1819 end 1812 1820 %create subdirectory for executable files … … 1840 1848 end 1841 1849 for ilist=1:size(Param.InputTable,1) 1842 Param.InputTable{ilist,1}=regexprep(Param.InputTable{ilist,1},'\','/');%correct path name for PCWIN system1850 Param.InputTable{ilist,1}=regexprep(Param.InputTable{ilist,1},'\','/');%correct path name for PCWIN system 1843 1851 end 1844 1852 % create, fill and save the xml parameter file 1845 1853 t=struct2xml(Param); 1846 1854 t=set(t,1,'name','Series'); 1847 filexml =fullfile_uvmat(DirXml,'',Param.InputTable{1,3},'.xml',OutputNomType,...1855 filexml{iprocess}=fullfile_uvmat(DirXml,'',Param.InputTable{1,3},'.xml',OutputNomType,... 1848 1856 Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j); 1849 save(t,filexml );% save the parameter file1857 save(t,filexml{iprocess});% save the parameter file 1850 1858 1851 %create the executable file 1852 filebat=fullfile_uvmat(DirExe,'',Param.InputTable{1,3},ExeExt,OutputNomType,... 1853 Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j); 1854 batch_file_list{iprocess}=filebat; 1859 1860 % set the log file name 1861 filelog{iprocess}=fullfile_uvmat(DirLog,'',Param.InputTable{1,3},'.log',OutputNomType,... 1862 Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j); 1863 end 1864 end 1865 1866 %% launch the executable files for background or cluster processing 1867 switch RunMode 1868 case 'background' 1869 % create the executable file 1870 filebat=fullfile_uvmat(DirExe,'',Param.InputTable{1,3},ExeExt,OutputNomType,... 1871 first_i,last_i,first_j,last_j); 1872 %batch_file_list{iprocess}=filebat; 1855 1873 [fid,message]=fopen(filebat,'w');% create the executable file 1856 1874 if isequal(fid,-1) … … 1858 1876 return 1859 1877 end 1860 1861 % set the log file name1862 1878 filelog=fullfile_uvmat(DirLog,'',Param.InputTable{1,3},'.log',OutputNomType,... 1863 Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j); 1864 1865 % fill and save the executable file 1879 first_i,last_i,first_j,last_j); 1866 1880 switch ActionExt 1867 1881 case '.m'% Matlab function … … 1873 1887 'matlab -nodisplay -nosplash -nojvm -logfile ''' filelog ''' <<END_MATLAB \n'... 1874 1888 'addpath(''' path_series '''); \n'... 1875 'addpath(''' Param.Action.ActionPath '''); \n'... 1876 '' Param.Action.ActionName '( ''' filexml '''); \n'... 1877 'exit \n'... 1878 'END_MATLAB \n']; 1889 'addpath(''' Param.Action.ActionPath '''); \n']; 1890 for iprocess=1:NbProcess 1891 cmd=[cmd '' Param.Action.ActionName '( ''' filexml{iprocess} '''); \n']; 1892 end 1893 cmd=[cmd 'exit \n' 'END_MATLAB \n']; 1879 1894 fprintf(fid,cmd);%fill the executable file with the char string cmd 1880 1895 fclose(fid);% close the executable file 1881 1896 system(['chmod +x ' filebat]);% set the file to executable 1882 1897 case {'PCWIN','PCWIN64'} 1883 text_matlabscript=['matlab -automation -logfile ' regexprep(filelog,'\\','\\\\')...1898 cmd=['matlab -automation -logfile ' regexprep(filelog{iprocess},'\\','\\\\')... 1884 1899 ' -r "addpath(''' regexprep(path_series,'\\','\\\\') ''');'... 1885 'addpath(''' regexprep(Param.Action.ActionPath,'\\','\\\\') ''');'... 1886 '' Param.Action.ActionName '( ''' regexprep(filexml,'\\','\\\\') ''');exit"']; 1887 fprintf(fid,text_matlabscript);%fill the executable file with the char string cmd 1900 'addpath(''' regexprep(Param.Action.ActionPath,'\\','\\\\') ''');']; 1901 for iprocess=1:NbProcess 1902 cmd=[cmd '' Param.Action.ActionName '( ''' regexprep(filexml{iprocess},'\\','\\\\') ''');'] 1903 end 1904 cmd=[cmd ';exit"']; 1905 fprintf(fid,cmd);%fill the executable file with the char string cmd 1888 1906 fclose(fid);% close the executable file 1889 1907 end 1908 system([filebat ' &'])% directly execute the command file 1890 1909 case '.sh' % compiled Matlab function 1891 1910 switch computer … … 1895 1914 'sleep $(( $RANDOM / 3000 )) \n '... 1896 1915 'hostname && date \n '... 1897 'umask 002 \n'... 1898 fullfile(ActionPath,[ActionName '.sh']) ' ' RunTime ' ' filexml];%allow writting access to created files for user group 1916 'umask 002 \n'] 1917 for iprocess=1:NbProcess 1918 cmd=[cmd fullfile(ActionPath,[ActionName '.sh']) ' ' RunTime ' ' filexml{iprocess}];%allow writting access to created files for user group 1919 end 1899 1920 fprintf(fid,cmd);%fill the executable file with the char string cmd 1900 1921 fclose(fid);% close the executable file 1901 1922 system(['chmod +x ' filebat]);% set the file to executable 1902 1903 case {'PCWIN','PCWIN64'} 1904 fprintf(fid,cmd);1905 fclose(fid);1923 system([filebat ' &'])% directly execute the command file 1924 case {'PCWIN','PCWIN64'} 1925 msgbox_uvmat('ERROR','option for compiled Matlab functions not implemented for Windows system') 1926 return 1906 1927 end 1907 end 1908 end 1909 end 1910 1911 %% launch the executable files for background or cluster processing 1912 switch RunMode 1913 case 'background' 1914 for iprocess=1:NbProcess 1915 system([batch_file_list{iprocess} ' &'])% directly execute the command file for each process 1916 end 1917 msgbox_uvmat('CONFIRMATION',[ActionName ' launched in background: press STATUS to see results']) 1918 case 'cluster_oar' % option 'oar-parexec' used 1919 %create subdirectory for oar command and log files 1920 %DirOARLog=fullfile(OutputDir,'0_LOG'); 1921 DirOAR=fullfile(OutputDir,'0_OAR'); 1922 if exist(DirOAR,'dir')% delete the content of the dir 0_LOG to allow new input 1923 curdir=pwd; 1924 cd(DirOAR) 1925 delete('*') 1926 cd(curdir) 1927 else 1928 [tild,msg1]=mkdir(DirOAR); 1929 if ~strcmp(msg1,'') 1930 errormsg=['cannot create ' DirOAR ': ' msg1];%error message for directory creation 1931 return 1932 end 1933 end 1934 filename_joblist=fullfile(DirOAR,'job_list.txt');%create name of the global executable file 1935 filename_log=fullfile(DirLog,'job_list.stdout');%file for output messages of the master oar process 1936 filename_errors=fullfile(DirLog,'job_list.stderr');%file for error messages of the master oar process 1937 1938 fid=fopen(filename_joblist,'w'); 1939 for p=1:length(batch_file_list) 1940 fprintf(fid,[batch_file_list{p} '\n']);% list of exe files 1941 end 1942 fclose(fid); 1943 system(['chmod +x ' filename_joblist]);% set the file to executable 1944 1945 % the command job_list.txt contains the list of NbProcess independent individual jobs 1946 % in which the total calculation has been split. Those are written as executable files .sh in the folder /O_EXE. 1947 % These individual jobs are grouped by the system as oar jobs on the NbCore processors. 1948 % For each processor, the oar job must stop after the walltime which has been set, which is limited to 24 h. 1949 % However, the oar job is automatically restarted (option 'idempotent') provided the individual jobs are 1950 % shorter than the wall time: in the time interval 'checkpoint' (WallTimeOneJob) before the end of the allowed duration, 1951 % the oar job restarts when an individual job ends. 1952 JobTime=CPUTime*BlockLength*nbfield_j;% estimated time for one individual job (in minutes) 1953 % wall time (in hours ) for each oar job, allowing 10 individual jobs, but limited to 23 h: 1954 WallTimeTotal=min(23,4*JobTime/60); 1955 %disp(['WallTimeTotal: ' num2str(WallTimeTotal) ' hours']) 1956 % estimated time of an individual job (in min), with a margin of error 1957 WallTimeOneJob=min(4*JobTime+10,WallTimeTotal*60/2);% estimated max time of an individual job for checkpoint 1958 disp(['WallTimeOneJob: ' num2str(WallTimeOneJob) ' minutes']) 1959 oar_command=['oarsub -n UVmat_' ActionName ' '... 1960 '-t idempotent --checkpoint ' num2str(WallTimeOneJob*60) ' '... 1961 '-l /core=' num2str(NbCore) ','... 1962 'walltime=' datestr(WallTimeTotal/24,13) ' '... 1963 '-E ' filename_errors ' '... 1964 '-O ' filename_log ' '... 1965 extra_oar ' '... 1966 '"oar-parexec -s -f ' filename_joblist ' '... 1967 '-l ' filename_joblist '.log"']; 1968 fprintf(oar_command);% display system command on the Matlab command window 1969 [status,result]=system(oar_command)% execute system command and show the result (ID number of the launched job) on the Matlab command window 1970 filename_oarcommand=fullfile(DirOAR,'0_oar_command');% keep track of the command in file '0-OAR/0_oar_command' 1971 fid=fopen(filename_oarcommand,'w'); 1972 fprintf(fid,oar_command); % store the command 1973 fprintf(fid,result);% store the result (job ID number) 1974 fclose(fid); 1975 1976 msgbox_uvmat('CONFIRMATION',[ActionName ' launched as ' num2str(NbProcess) ' processes in cluster: press STATUS to see results']) 1977 case 'cluster_pbs' % for LMFA Kepler machine 1978 %create subdirectory for pbs command and log files 1979 DirPBS=fullfile(OutputDir,'0_PBS'); %todo : common name OAR/PBS 1980 if exist(DirPBS,'dir')% delete the content of the dir 0_LOG to allow new input 1981 curdir=pwd; 1982 cd(DirPBS) 1983 delete('*') 1984 cd(curdir) 1985 else 1986 [tild,msg1]=mkdir(DirPBS); 1987 if ~strcmp(msg1,'') 1988 errormsg=['cannot create ' DirPBS ': ' msg1];%error message for directory creation 1989 return 1990 end 1991 end 1992 max_walltime=3600*20; % 20h max total calculation (cannot exceed 24 h) 1993 walltime_onejob=1800; % seconds, max estimated time for asingle file index value 1994 filename_joblist=fullfile(DirPBS,'job_list.txt');%create name of the global executable file 1995 fid=fopen(filename_joblist,'w'); 1996 for p=1:length(batch_file_list) 1997 fprintf(fid,[batch_file_list{p} '\n']);% list of exe files 1998 end 1999 fclose(fid); 2000 system(['chmod +x ' filename_joblist]);% set the file to executable 2001 pbs_command=['qstat -n CIVX '... 2002 '-t idempotent --checkpoint ' num2str(walltime_onejob+60) ' '... 2003 '-l /core=' num2str(NbCore) ','... 2004 'walltime=' datestr(min(1.05*walltime_onejob/86400*max(NbProcess*BlockLength*nbfield_j,NbCore)/NbCore,max_walltime/86400),13) ' '... 2005 '-E ' regexprep(filename_joblist,'\.txt\>','.stderr') ' '... 2006 '-O ' regexprep(filename_joblist,'\.txt\>','.log') ' '... 2007 extra_oar ' '... 2008 '"oar-parexec -s -f ' filename_joblist ' '... 2009 '-l ' filename_joblist '.log"']; 2010 filename_oarcommand=fullfile(DirPBS,'pbs_command'); 2011 fid=fopen(filename_oarcommand,'w'); 2012 fprintf(fid,pbs_command); 2013 fclose(fid); 2014 fprintf(pbs_command);% display in command line 2015 %system(pbs_command); 2016 msgbox_uvmat('CONFIRMATION',[ActionName ' command ready to be launched in cluster']) 2017 case 'python' 2018 command = [ 2019 'LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | pyp "p.split('':'') | [s for s in p if ''matlab'' not in s] | '':''.join(p)") ' ... 2020 'python -m fluiddyn.postproc.uvmat ' filexml]; 2021 % fprintf(['command:\n' command '\n\n']) 2022 system(command, '-echo'); 2023 end 2024 1928 msgbox_uvmat('CONFIRMATION',[ActionName ' launched in background: press STATUS to see results']) 1929 case 'cluster_oar' % option 'oar-parexec' used 1930 %create subdirectory for oar command and global log files 1931 DirOAR=fullfile(OutputDir,'0_OAR'); 1932 if exist(DirOAR,'dir')% delete the content of the dir 0_LOG to allow new input 1933 curdir=pwd; 1934 cd(DirOAR) 1935 delete('*') 1936 cd(curdir) 1937 else 1938 [tild,msg1]=mkdir(DirOAR); 1939 if ~strcmp(msg1,'') 1940 errormsg=['cannot create ' DirOAR ': ' msg1];%error message for directory creation 1941 return 1942 end 1943 end 1944 filename_joblist=fullfile(DirOAR,'job_list.txt');%create name of the global executable file 1945 filename_log=fullfile(DirLog,'job_list.stdout');%file for output messages of the master oar process 1946 filename_errors=fullfile(DirLog,'job_list.stderr');%file for error messages of the master oar process 1947 1948 fid=fopen(filename_joblist,'w'); 1949 for p=1:length(batch_file_list) 1950 fprintf(fid,[batch_file_list{p} '\n']);% list of exe files 1951 end 1952 fclose(fid); 1953 system(['chmod +x ' filename_joblist]);% set the file to executable 1954 1955 % the command job_list.txt contains the list of NbProcess independent individual jobs 1956 % in which the total calculation has been split. Those are written as executable files .sh in the folder /O_EXE. 1957 % These individual jobs are grouped by the system as oar jobs on the NbCore processors. 1958 % For each processor, the oar job must stop after the walltime which has been set, which is limited to 24 h. 1959 % However, the oar job is automatically restarted (option 'idempotent') provided the individual jobs are 1960 % shorter than the wall time: in the time interval 'checkpoint' (WallTimeOneJob) before the end of the allowed duration, 1961 % the oar job restarts when an individual job ends. 1962 JobTime=CPUTime*BlockLength*nbfield_j;% estimated time for one individual job (in minutes) 1963 % wall time (in hours ) for each oar job, allowing 10 individual jobs, but limited to 23 h: 1964 WallTimeTotal=min(23,4*JobTime/60); 1965 %disp(['WallTimeTotal: ' num2str(WallTimeTotal) ' hours']) 1966 % estimated time of an individual job (in min), with a margin of error 1967 WallTimeOneJob=min(4*JobTime+10,WallTimeTotal*60/2);% estimated max time of an individual job for checkpoint 1968 disp(['WallTimeOneJob: ' num2str(WallTimeOneJob) ' minutes']) 1969 oar_command=['oarsub -n UVmat_' ActionName ' '... 1970 '-t idempotent --checkpoint ' num2str(WallTimeOneJob*60) ' '... 1971 '-l /core=' num2str(NbCore) ','... 1972 'walltime=' datestr(WallTimeTotal/24,13) ' '... 1973 '-E ' filename_errors ' '... 1974 '-O ' filename_log ' '... 1975 extra_oar ' '... 1976 '"oar-parexec -s -f ' filename_joblist ' '... 1977 '-l ' filename_joblist '.log"']; 1978 fprintf(oar_command);% display system command on the Matlab command window 1979 [status,result]=system(oar_command)% execute system command and show the result (ID number of the launched job) on the Matlab command window 1980 filename_oarcommand=fullfile(DirOAR,'0_oar_command');% keep track of the command in file '0-OAR/0_oar_command' 1981 fid=fopen(filename_oarcommand,'w'); 1982 fprintf(fid,oar_command); % store the command 1983 fprintf(fid,result);% store the result (job ID number) 1984 fclose(fid); 1985 1986 msgbox_uvmat('CONFIRMATION',[ActionName ' launched as ' num2str(NbProcess) ' processes in cluster: press STATUS to see results']) 1987 case 'cluster_pbs' % for LMFA Kepler machine 1988 %create subdirectory for pbs command and log files 1989 DirPBS=fullfile(OutputDir,'0_PBS'); %todo : common name OAR/PBS 1990 if exist(DirPBS,'dir')% delete the content of the dir 0_LOG to allow new input 1991 curdir=pwd; 1992 cd(DirPBS) 1993 delete('*') 1994 cd(curdir) 1995 else 1996 [tild,msg1]=mkdir(DirPBS); 1997 if ~strcmp(msg1,'') 1998 errormsg=['cannot create ' DirPBS ': ' msg1];%error message for directory creation 1999 return 2000 end 2001 end 2002 max_walltime=3600*20; % 20h max total calculation (cannot exceed 24 h) 2003 walltime_onejob=1800; % seconds, max estimated time for asingle file index value 2004 filename_joblist=fullfile(DirPBS,'job_list.txt');%create name of the global executable file 2005 fid=fopen(filename_joblist,'w'); 2006 for p=1:length(batch_file_list) 2007 fprintf(fid,[batch_file_list{p} '\n']);% list of exe files 2008 end 2009 fclose(fid); 2010 system(['chmod +x ' filename_joblist]);% set the file to executable 2011 pbs_command=['qstat -n CIVX '... 2012 '-t idempotent --checkpoint ' num2str(walltime_onejob+60) ' '... 2013 '-l /core=' num2str(NbCore) ','... 2014 'walltime=' datestr(min(1.05*walltime_onejob/86400*max(NbProcess*BlockLength*nbfield_j,NbCore)/NbCore,max_walltime/86400),13) ' '... 2015 '-E ' regexprep(filename_joblist,'\.txt\>','.stderr') ' '... 2016 '-O ' regexprep(filename_joblist,'\.txt\>','.log') ' '... 2017 extra_oar ' '... 2018 '"oar-parexec -s -f ' filename_joblist ' '... 2019 '-l ' filename_joblist '.log"']; 2020 filename_oarcommand=fullfile(DirPBS,'pbs_command'); 2021 fid=fopen(filename_oarcommand,'w'); 2022 fprintf(fid,pbs_command); 2023 fclose(fid); 2024 fprintf(pbs_command);% display in command line 2025 %system(pbs_command); 2026 msgbox_uvmat('CONFIRMATION',[ActionName ' command ready to be launched in cluster']) 2027 case 'python' 2028 command = [ 2029 'LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | pyp "p.split('':'') | [s for s in p if ''matlab'' not in s] | '':''.join(p)") ' ... 2030 'python -m fluiddyn.postproc.uvmat ' filexml{iprocess}]; 2031 % fprintf(['command:\n' command '\n\n']) 2032 system(command, '-echo'); 2033 end 2034 end 2025 2035 %------------------------------------------------------------------------ 2026 2036 function STOP_Callback(hObject, eventdata, handles) -
trunk/src/series/sub_background.m
r908 r917 86 86 %% root input file(s) and type 87 87 % check the existence of the first file in the series 88 first_j=[]; 89 if isfield(Param.IndexRange,' first_j'); first_j=Param.IndexRange.first_j; end88 first_j=[];% note that the function will propose to cover the whole range of indices 89 if isfield(Param.IndexRange,'MinIndex_j'); first_j=Param.IndexRange.MinIndex_j; end 90 90 last_j=[]; 91 if isfield(Param.IndexRange,' last_j'); last_j=Param.IndexRange.last_j; end91 if isfield(Param.IndexRange,'MaxIndex_j'); last_j=Param.IndexRange.MaxIndex_j; end 92 92 PairString=''; 93 93 if isfield(Param.IndexRange,'PairString'); PairString=Param.IndexRange.PairString; end … … 131 131 end 132 132 first_i=1;last_i=1;incr_i=1;%default 133 if isfield(Param.IndexRange,' first_i'); last_i=Param.IndexRange.first_i; end134 if isfield(Param.IndexRange,' last_i'); last_i=Param.IndexRange.last_i; end133 if isfield(Param.IndexRange,'MinIndex_i'); first_i=Param.IndexRange.MinIndex_i; end 134 if isfield(Param.IndexRange,'MaxIndex_i'); last_i=Param.IndexRange.MaxIndex_i; end 135 135 if isfield(Param.IndexRange,'incr_i')&&~isempty(Param.IndexRange.incr_i) 136 136 incr_i=Param.IndexRange.incr_i; … … 156 156 def = { 'No';num2str(nbaver_init);'0.1'}; 157 157 answer = inputdlg(prompt,dlg_title,num_lines,def); 158 158 if isempty(answer) 159 return 160 end 159 161 %check input consistency 160 162 if strcmp(answer{1},'No') && ~isequal(NbSlice_i,1) … … 252 254 nbfield_j=size(i1_series{1},1); %nb of fields for the j index (bursts or volume slices) 253 255 nbfield_i=size(i1_series{1},2); %nb of fields for the i index 254 nbfield=nbfield_j*nbfield_i; %total number of fields 256 255 257 if Param.ActionInput.CheckVolume 256 258 step=2;% we assume the burst contains only one image pair … … 262 264 NbSlice_j=1; 263 265 NbSlice=NbSlice_i; 264 nbfield_i=floor(nbfield/NbSlice_i);%total number of indexes in a slice (adjusted to an integer number of slices) 265 nbfield=nbfield_i*NbSlice_i; %total number of fields after adjustement 266 nbfield_series=nbfield; 267 end 266 %nbfield_i=floor(nbfield_i/NbSlice_i);%total number of indexes in a slice (adjusted to an integer number of slices) 267 %nbfield=nbfield_i*NbSlice_i; %total number of fields after adjustement 268 nbfield_series=nbfield_i*nbfield_j; 269 end 270 nbfield=nbfield_j*nbfield_i; %total number of fields 268 271 nbaver_ima=Param.ActionInput.SlidingSequenceLength;%number of images for the sliding background 269 272 nbaver=ceil(nbaver_ima/step);%number of bursts for the sliding background
Note: See TracChangeset
for help on using the changeset viewer.
