source: trunk/project-meta/project-meta @ 196

Last change on this file since 196 was 196, checked in by g7moreau, 6 years ago
  • Add module version
  • Property svn:executable set to *
File size: 14.7 KB
Line 
1#!/usr/bin/env perl
2#
3# 2018/01/17 Gabriel Moreau <Gabriel.Moreau(A)univ-grenoble-alpes.fr>
4#
5# apt-get install yamllint libyaml-syck-perl libtemplate-perl libarchive-zip-perl
6
7use strict;
8use warnings;
9use version; our $VERSION = version->declare('0.0.3');
10
11use File::Copy qw(copy);   
12use YAML::Syck;
13use Getopt::Long();
14use Cwd();
15use Template;
16use Archive::Zip qw(:ERROR_CODES :CONSTANTS);
17
18
19my ($verbose);
20Getopt::Long::GetOptions(
21   'verbose' => \$verbose,
22   );
23
24
25my %CMD_DB = (
26   'help'            => \&cmd_help,
27   'version'         => \&cmd_version,
28   'check'           => \&cmd_check,
29   'make-link'       => \&cmd_make_link,
30   'remove-link'     => \&cmd_remove_link,
31   'make-zip'        => \&cmd_make_zip,
32   'make-author'     => \&cmd_make_author,
33   'make-licence'    => \&cmd_make_licence,
34   'make-copyright'  => \&cmd_make_copyright,
35   'list-licence'    => \&cmd_list_licence,
36   );
37
38################################################################
39# main program
40################################################################
41
42my $cmd = shift @ARGV || 'help';
43if (defined $CMD_DB{$cmd}) {
44   $CMD_DB{$cmd}->(@ARGV);
45   }
46else {
47   print {*STDERR} "project-meta: command $cmd not found\n\n";
48   $CMD_DB{'help'}->();
49   exit 1;
50   }
51
52exit;
53
54################################################################
55# subroutine
56################################################################
57
58sub print_ok {
59   my ($key, $test) = @_;
60   
61   printf "%-35s : %s\n", $key, $test ? 'yes' : 'no';
62   }
63
64################################################################
65
66sub addfolder2list {
67   my ($folderdb, $folder) = @_;
68   
69   $folder =~ s{/[^/]+$}{};
70   
71   return if $folder !~ m{/};
72
73   $folderdb->{$folder}++;
74   return addfolder2list($folderdb, $folder);
75   }
76
77################################################################
78# command
79################################################################
80
81sub cmd_help {
82   print <<'END';
83project-meta - opendata project metafile manager
84
85 project-meta help
86 project-meta version
87 project-meta check
88 project-meta make-link
89 project-meta remove-link
90 project-meta make-zip
91 project-meta make-author
92 project-meta make-licence
93 project-meta make-copyright
94 project-meta make-licence
95 project-meta list-licence
96END
97   }
98
99################################################################
100
101sub cmd_version {
102   print "$VERSION\n";
103   }
104
105################################################################
106
107sub cmd_check {
108   my $meta = YAML::Syck::LoadFile("PROJECT-META.yml");
109
110   my $acronym     = $meta->{'project'}{'acronym'};
111   my $current_dir = Cwd::getcwd();
112   my $dap_folder  = $meta->{'public-dap'}{'dap-folder'};
113
114   print_ok 'project/acronym',                  $acronym =~ m{\d\d\w[\w\d_/]+};
115   print_ok 'public-dap/dap-folder',            $dap_folder ne '' and $dap_folder =~ m{^/};
116   print_ok 'dap-folder not match current_dir', $dap_folder !~ m{$current_dir};
117
118   #print YAML::Syck::Dump($meta);
119   }
120
121################################################################
122
123sub cmd_make_link {
124   my $meta = YAML::Syck::LoadFile("PROJECT-META.yml");
125   my $current_dir = Cwd::getcwd();
126   my $acronym     = $meta->{'project'}{'acronym'};
127   my $dap_folder  = $meta->{'public-dap'}{'dap-folder'};
128   my $data_set    = $meta->{'public-dap'}{'data-set'};
129
130   push @{$data_set}, 'AUTHORS.txt', 'COPYRIGHT.txt', 'LICENCE.txt';
131   {
132      # Remove doublon
133      my %seen = ();
134      @{$data_set} = grep { ! $seen{$_}++ } @{$data_set};
135      }
136
137   # Create a list of the folder
138   my %folders;
139   for my $dataset (@{$data_set}) {
140      addfolder2list(\%folders, $dataset);
141      }
142
143   print "chmod o+rX,o-w '$current_dir'\n";
144   print "mkdir -p '$dap_folder/$acronym'\n" if not -d "$dap_folder/$acronym";
145   for my $folder (sort keys %folders) {
146      print "chmod o+rX,o-w '$current_dir/$folder'\n";
147      print "mkdir -p '$dap_folder/$acronym/$folder'\n" if -d "$current_dir/$folder";
148      }
149
150   for my $dataset (@{$data_set}) {
151      if ($dataset =~ m{/}) {
152         # Folder case
153         my $folder = $dataset =~ s{/[^/]+$}{}r;
154         print "ln --symbolic --target-directory '$dap_folder/$acronym/$folder/' '$current_dir/$dataset'\n";
155         }
156      else {
157         # File case
158         print "ln --symbolic --target-directory '$dap_folder/$acronym/' '$current_dir/$dataset'\n";
159         }
160
161      }
162   print "chmod -R o+rX,o-w '$dap_folder/$acronym/'\n";
163   }
164
165################################################################
166
167sub cmd_remove_link {
168   my $meta = YAML::Syck::LoadFile("PROJECT-META.yml");
169   my $current_dir = Cwd::getcwd();
170   my $acronym     = $meta->{'project'}{'acronym'};
171   my $dap_folder  = $meta->{'public-dap'}{'dap-folder'};
172
173   die "Error: DAP folder match current folder" if $dap_folder =~ m{$current_dir} or $current_dir =~ m{$dap_folder};
174
175   print "find '$dap_folder/$acronym/' -type l -o -type d -exec ls -l {} \+\n";
176   print "find '$dap_folder/$acronym/' -type l -delete\n";
177   print "find '$dap_folder/$acronym/' -type d -delete\n";
178   }
179
180################################################################
181
182sub cmd_make_zip {
183   my $meta = YAML::Syck::LoadFile("PROJECT-META.yml");
184   my $current_dir = Cwd::getcwd();
185   my $data_set    = $meta->{'public-dap'}{'data-set'};
186   my $acronym     = $meta->{'project'}{'acronym'};
187
188   push @{$data_set}, 'AUTHORS.txt', 'COPYRIGHT.txt', 'LICENCE.txt';
189   {
190      # Remove doublon
191      my %seen = ();
192      @{$data_set} = grep { ! $seen{$_}++ } @{$data_set};
193      }
194
195   # Create a Zip file
196   my $zip = Archive::Zip->new();
197
198   for my $dataset (@{$data_set}) {
199      if (-d $dataset) {
200         # Folder case
201         $zip->addTree($dataset, "$acronym/$dataset");
202         }
203      elsif (-f $dataset) {
204         # File case
205         $zip->addFile($dataset, "$acronym/$dataset");
206         }
207      else {
208         # Strange case
209         print "Error: entry $dataset doesn't exists\n";
210         }
211      }
212
213   my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime time;
214   $year += 1900;
215   $mon++;
216   my $date = sprintf '%04i%02i%02i-%02i%02i', $year, $mon, $mday, $hour, $min;
217
218   # Save the Zip file
219   unless ($zip->writeToFileNamed("$current_dir/$acronym--$date.zip") == AZ_OK) {
220      die 'Error: zip write error';
221      }
222   }
223
224################################################################
225
226sub cmd_make_author {
227   my $meta = YAML::Syck::LoadFile("PROJECT-META.yml");
228
229   my $current_dir = Cwd::getcwd();
230
231   my $acronym    = $meta->{'project'}{'acronym'};
232   my $authors_list = $meta->{'project'}{'authors'};
233
234   if (-f "$current_dir/AUTHORS.txt") {
235      # Test for manual or automatically generated file
236      # Automatically generated file by project-meta
237      my $automatic;
238      open my $fh, '<', "$current_dir/AUTHORS.txt" or die $!;
239      for my $line (<$fh>) {
240         $line =~ m/Automatically generated .* project-meta/i and $automatic++;
241         }
242      close $fh;
243
244      if (not $automatic) {
245         print "Warning: AUTHORS.txt already exists\n";
246         return;
247         }
248
249      print "Warning: update AUTHORS.txt\n";
250      }
251
252   my $tt = Template->new(INCLUDE_PATH => '/usr/share/project-meta/template.d');
253   my $msg_format = '';
254   $tt->process('AUTHORS.tt',
255      {
256         acronym    => $acronym,
257         authorlist => $authors_list,
258      }, \$msg_format) || die $tt->error;
259
260   open my $fh,  '>', "$current_dir/AUTHORS.txt" or die $!;
261   print $fh "$msg_format\n\n";
262   close $fh;
263   }
264
265################################################################
266
267sub cmd_make_licence {
268   my $meta = YAML::Syck::LoadFile("PROJECT-META.yml");
269
270   my $current_dir = Cwd::getcwd();
271
272   if (-f "$current_dir/LICENCE.txt") {
273      print "Warning: LICENCE.txt already exists\n";
274      return;
275      }
276
277   my $licence = $meta->{'public-dap'}{'data-licence'};
278
279   if (not -f "/usr/share/project-meta/licence.d/$licence.txt") {
280      print "Error: licence $licence doesn't exists in project-meta database\n";
281      exit 1;
282      }
283
284   copy("/usr/share/project-meta/licence.d/$licence.txt", "$current_dir/LICENCE.txt")
285      or die "Error: licence copy failed - $!";
286
287   print "Info: LICENCE.txt file create\n";
288   return;
289   }
290
291################################################################
292
293sub cmd_make_copyright {
294   my $meta = YAML::Syck::LoadFile("PROJECT-META.yml");
295
296   my $current_dir = Cwd::getcwd();
297
298   if (-f "$current_dir/COPYRIGHT.txt") {
299      # Test for manual or automatically generated file
300      # Automatically generated file by project-meta
301      my $automatic;
302      open my $fh, '<', "$current_dir/COPYRIGHT.txt" or die $!;
303      for my $line (<$fh>) {
304         $line =~ m/Automatically generated .* project-meta/i and $automatic++;
305         }
306      close $fh;
307
308      if (not $automatic) {
309         print "Warning: COPYRIGHT.txt already exists\n";
310         return;
311         }
312
313      print "Warning: update COPYRIGHT.txt\n";
314      }
315   
316   my $tt = Template->new(
317      INCLUDE_PATH   => '/usr/share/project-meta/template.d',
318      POST_CHOMP     => 1, # Remove space and carriage return after %]
319      );
320   my $msg_format = '';
321   $tt->process('COPYRIGHT.tt',
322      {
323         title       => $meta->{'project'}{'title'},
324         acronym     => $meta->{'project'}{'acronym'},
325         authorlist  => $meta->{'project'}{'authors'},
326         description => $meta->{'project'}{'short-description'},
327         licence     => $meta->{'public-dap'}{'data-licence'},
328         doi         => $meta->{'publication'}{'doi'},
329      }, \$msg_format) || die $tt->error;
330
331   open my $fh,  '>', "$current_dir/COPYRIGHT.txt" or die $!;
332   print $fh "$msg_format\n\n";
333   close $fh;
334   }
335
336################################################################
337
338sub cmd_list_licence {
339   opendir my $dh, '/usr/share/project-meta/licence.d/' or die $!;
340   for my $licence (readdir $dh) {
341      # Keep only file
342      next if not -f "/usr/share/project-meta/licence.d/$licence";
343     
344      # Keep only .txt file
345      next if not $licence =~ m/\.txt$/;
346
347      $licence =~ s/\.txt$//;
348      print "$licence\n";
349      }
350   closedir $dh;
351   }
352
353################################################################
354# documentation
355################################################################
356
357__END__
358
359=head1 NAME
360
361project-meta - opendata project metafile manager
362
363
364=head1 USAGE
365
366 project-meta help
367 project-meta version
368 project-meta check
369 project-meta make-link
370 project-meta remove-link
371 project-meta make-zip
372 project-meta make-author
373 project-meta make-licence
374 project-meta make-copyright
375 project-meta make-licence
376 project-meta list-licence
377
378=head1 DESCRIPTION
379
380Project-Meta is a small tool to maintain a set of open data files.
381In order to help you in this task, C<project-meta> command has a set of action
382to generated and maintain many files in your dataset.
383
384Everything is declare in the metafile F<PROJECT-META.yml>.
385This YAML file must exist in your root projet folder.
386See L</METAFILE SPECIFICATION>.
387
388=head1 COMMANDS
389
390Some command are defined in the source code but are not documented here.
391Theses could be not well defined, not finished, not well tested...
392You can read the source code and use them at your own risk
393(like for all the Project-Meta code).
394
395=head2 check
396
397 project-meta check
398
399Check your F<PROJECT-META.yml> has the good key.
400If your metafile is not a valid YAML file,
401you can use C<yamllint> command to check just it's format.
402
403=head2 make-link
404
405 project-meta make-link
406
407Create UNIX soft links on the OpeNDAP folder to the real data.
408Files F<AUTHORS.txt>, F<LICENCE.txt> and F<COPYRIGHT.txt> are mandatory but could be generated (see below).
409The main keys use in the F<PROJECT-META.yml> are:
410
411=over
412
413=item * C<project/acronym>: the project short acronym, add to the OpeNDAP root folder
414
415=item * C<public-dap/dap-folder>: the OpeNDAP root folder
416
417=item * C<public-dap/data-set>: a list of files or folder to push
418
419=back
420
421Because this command could be dangerous, it does nothing!
422It print on terminal shell command to be done.
423You have to verify ouput before eval it.
424
425 project-meta make-link
426 project-meta make-link | bash
427
428=head2 remove-link
429
430 project-meta remove-link
431
432Remove link in OpeNDAP folder for that projet.
433Because command C<rm> is always dangerous,
434we use here the command C<find> limited to folder and link.
435
436Please verify the returned values before excuted it with the C<-delete> option.
437
438=head2 make-zip
439
440 project-meta make-zip
441
442Create a ZIP archive with the open data set.
443Files F<AUTHORS.txt>, F<LICENCE.txt> and F<COPYRIGHT.txt> are mandatory but could be generated (see below).
444The main keys use in the F<PROJECT-META.yml> are:
445
446=over
447
448=item * C<project/acronym>: the project short acronym, use as root folder
449
450=item * C<public-dap/data-set>: a list of files or folder to push
451
452=back
453
454=head2 make-licence
455
456 project-meta make-licence
457
458Copy the licence file from the project-meta licence database at the current folder
459with the file name: F<LICENCE.txt>.
460
461The licence is defined in the F<PROJECT-META.yml> specification under the key C<public-dap/data-licence>.
462The list of possible licence is given with the command L</list-licence>.
463
464=head2 list-licence
465
466 project-meta list-licence
467
468Give the list of all the open data licence supported by the project-meta licence database.
469At this time the possible licence are:
470
471 license-ouverte-v2.0
472 open-database-license-v1.0
473
474Note that these licences are dedicated to open data.
475Please do not use open licence that have been written for code or documentation for data.
476
477=head2 make-author
478
479 project-meta make-author
480
481Create or update the F<AUTHORS.txt> file at the current folder.
482Authors data are extracted from the C<PROJECT-META.yml> file.
483
484=head2 make-copyright
485
486 project-meta make-copyright
487
488Create or update the F<COPYRIGHT.txt> file at the current folder.
489Authors, licence and copyright data are extracted from the C<PROJECT-META.yml> file.
490
491
492=head1 METAFILE SPECIFICATION
493
494Each project must have an open data metafile which describe the project : C<PROJECT-META.yml>.
495The file is in YAML format because this is a human readable style of text file.
496
497You can find in the project-meta software a C<PROJECT-META.sample.yml> example.
498This one is actually the master reference specification!
499
500
501=head1 KNOWN BUGS
502
503 - not really check keys and tags before doing action!
504
505=head1 SEE ALSO
506
507yamllint
508
509
510=head1 AUTHOR
511
512Written by Gabriel Moreau, LEGI UMR5519, CNRS, Grenoble - France
513
514=head1 SPECIAL THANKS
515
516The list of people below did not directly contribute to project-meta's source code
517but provided me with some data, returned bugs
518or helped me in another task like having new ideas, specifications...
519Maybe I forgot your contribution in recent years,
520please forgive me in advance and send me an e-mail to correct this.
521
522Joel Sommeria, Julien Chauchat, Cyrille Bonamy, Antoine Mathieu.
523
524
525=head1 LICENSE AND COPYRIGHT
526
527Licence GNU GPL version 2 or later and Perl equivalent
528
529Copyright (C) 2017-2018 Gabriel Moreau <Gabriel.Moreau(A)univ-grenoble-alpes.fr>.
Note: See TracBrowser for help on using the repository browser.