From e7e95cfe51acd13115a07d2d21e1b67c50f3548d Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 20 May 2026 10:59:52 +0200 Subject: [PATCH 1/3] Skip files that cannot be accessed. --- src/cso/cso_colhub.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/cso/cso_colhub.py b/src/cso/cso_colhub.py index 35c4984..539225d 100644 --- a/src/cso/cso_colhub.py +++ b/src/cso/cso_colhub.py @@ -32,6 +32,9 @@ # Added production time of data files to listings. # Select file with latest production time if multiple are found. # +# 2026-04, Arjo Segers +# Skip files that cannot be accessed, for example a broken link. +# ######################################################################## ### @@ -209,13 +212,22 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): for fname in files: # data file? if fnmatch.fnmatch(fname, fpattern): - # already in table? - if fname in listing: + # full path: + href = os.path.join(root, fname) + #~ sometimes links to non-existing targets ... + if not os.path.isfile(href): + # info ... + logging.info(f"{indent} WARNING - invalid file (broken link?): {href}") + # + #~ already in table? + elif fname in listing: # info ... - logging.info(f"{indent} keep entry %s ..." % fname) + logging.info(f"{indent} keep entry {fname} ...") + # + #~ new entry: else: # info ... - logging.info(f"{indent} add entry %s ..." % fname) + logging.info(f"{indent} add entry {fname} ...") # Example filename: # S5P_RPRO_L2__CH4____20180430T001851_20180430T020219_02818_01_010301_20190513T141133.nc # @@ -270,7 +282,7 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): data["collection"] = collection data["processor_version"] = processor_version data["production_time"] = production_time - data["href"] = os.path.join(root, fname) + data["href"] = href # update record: listing.UpdateRecord(fname, data, indent=f"{indent} ") # endif # new record? -- GitLab From 03751be578a55a37f583818c2eecba192d669c09 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Thu, 18 Jun 2026 16:23:39 +0200 Subject: [PATCH 2/3] Added class to plot number of available data files per month. --- .../NO2/Copernicus_S5p_NO2_inquire-avail.png | Bin 0 -> 16724 bytes ...> Copernicus_S5p_NO2_inquire-versions.png} | Bin doc/source/s5p-no2.rst | 2 +- doc/source/tutorial.rst | 2 +- src/cso/cso_colhub.py | 2 +- src/cso/cso_file.py | 4 +- src/cso/cso_inquire.py | 267 +++++++++++++++++- 7 files changed, 264 insertions(+), 13 deletions(-) create mode 100644 doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-avail.png rename doc/source/figs/NO2/{Copernicus_S5p_NO2.png => Copernicus_S5p_NO2_inquire-versions.png} (100%) diff --git a/doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-avail.png b/doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-avail.png new file mode 100644 index 0000000000000000000000000000000000000000..171bcc0f30fbf52a0e5be27cd54e37daafbe1f00 GIT binary patch literal 16724 zcmeAS@N?(olHy`uVBq!ia0y~yVA;UHz&L?}je&t--`1IN3=9lbC9V-A!TD(=<%vb9 z48Dma1v&X8IhjccWvNBQnfZCfdWHsimO2UhXbB2jv*Cu-rlXO3Ay_B|Htns>JCCTg{BEOew)k2&Aj!5 zl&)xNh@j??T}+9~SA=X^pySFd#t|{K>DnTv&?wDKt-cBkE)#TFTec*eT;#+NEjH=* z@5AxWUYkhzBs_b@S7|HXUuk3Xea_DB#chWf^3T~O8}l3%U;u-JKs(V`FpGhqK$#^1 z!rWlOqyu3^NHK~*SUSB7+z{3@w+2=S%Uk&X6PVSoR4{=N%sOz1V*{AMkdQgdjQhO$ z1=E?3kdP(C&(E!_`ub|q%^cBKKCovUmL^NI>Ba6^QuX!K$>Ym&vR7+`Hk7`*W9jPV z7FHq&R>Ls$-p$SFR~I-o8+~0H!Wj1T)m2r`%*@PHfs5T(!3Xg@l3* zUOgZ!A|j$;VG*IIs8}Pg*axZu)quRP%5CLI!sQTGVU6LimEWK-`(xpvMM_VfKMyaF z+%8&QU;p#huU%i~Kx1}6(5dt1!=s{Rb#--V?V57iFCt=wl9JM?t%1LL1t1Bc!RxpI z&(h_~PhYsQ6? zt>WV2TlbG z@w5i!v`a)Z?qC;}79}O6EhR65uC9-dpKX?VY4drz=w-!oY^%eB)%~vQD12;E{VnHs zrMlmo6%7pxy^_Xmpira44)Uhe&VZ}-txr~AN;Wnf^4OL=>1>#DG|vp_MuXOGR# zA3riaK03Pe+uo|LtCso9REm#}FMWS+@0sTog^yaCoSj3r=iL>0&cneG@c3A7>7yf@ z-}%q6!pbN8ON-t4bz^shgs+Ra*u={Hb^uB*bdww$)phgd&tD(B+;3&%=CpZ|37W7XZ_kwe{{Fh} zyYFY7nxeU@?Cq?_6)r9+An|FZY{! z>Cey4U+)y3&wO;Gb6G9(LKkSM{NU=JKYvWJu4tT{ZSEhwE~fCd<*{Dr<^A^mBIehA zn`u@0YKd9yt&FFqrbg{5Ss7RRHME@X;qlkTU$Y>#i>3LPYwc>0j@p>yy7G8@#Y5J; z9}aO(QaQZiqr|DAzNEjuzlZ<-_xE({^4%MmXj~;cNIf^tcD4Gv3a78Ht_C|eG%Q-Q z$fWE|MD6#x<$ZSF?^UnQF^ldz{`K|s(|7Fh?(FE2h~un-RcH;VYqrd+-g~^KX#bNh zFE4Mp{Wj~`nwj^i-`kpIUGZ@5m%F;ebMlv$%jajEpJ%%&aB-W;%a_aNUpq6?cy8SL z=iloKFZ6GXZvCnSt>U~te*DPA!ExbOuXN~AFHz5+!)?4*HzvE^ExjHKikGb=FM~XT z3JNwTDJjLwdwu-Y+NNj@NTs;v$j{Huwy@#*Bp4^!-O+Jjf}-=6gY5Dr3{LIe zbovxL1syp#Sv^&vEoxg%WLEX*X}YBk53$yMK5IT{iK_Rs6;WHW`i>vz6rTI*f5h(4 z0&Ycclr>D(e9g65(Yfu#QStZ^8-8wX?eN{9&;ppDqEpRx)`_&unx>|q%xpXw7w65I zwaWV44(F}e*QcqpwzaMDoUEpI{(JJngPSToCOv)nR8vjO&ARN(g`1nxx7Piw`ug&+``6dk zudj*Re5Y(}{=T1X?EG>iRVis{yWajub%mu?*I8z{tN#A}zPj-7v8pE%-Ak^=mcN{t zJ}dqD5Ph zj&@0ZySXiQw)^d~M_^?y*7^<6f-zP-Fv~ExEyry3<;AYuVtwE4+=)4Q{P^m$v$H^j zR8n%Xca7@fna1f?*2mkkva_%Dn`<@8By-Zb`2BfqKOv#D!eLo0cjD`7YqwT@e)fO& zhWr0xtN#4>`0Kg-|CLu)hijXdge-RJy>y_Fd29A{J>9>*I{m z&IC9(G(;YTSg@TjK0e+vsIG3`s*sgI2e00#TNk%i%k$Is@8zd|`fzXXn`Z-Rz|uE2HeSho_VlTyzW(`-HtOofi-o1OpU z{d;|R`E8u#?1v3XzP<`gO-Sg*?H;m<*5@UEU13B z)7`yKCeY2Tjfsg#L`>}2msuY^6r`r4cuX=fGMaN}^`mc(%)+0YnYp;7h2=XRs3Fh5 zP+>d!?6O^@ueEe_egFSFU$3=m4yefzv9kQpU7cX2HNW5QzyGm#ciG!3JB!m@U0pM; zt_qclU1nbSDaCh&!9kGgK@Bgt-Ev?Le2`;gWX!s?W#y;OpS5>Qx!tvY|Nh%yaiYd= z#SY1HadQW6PCNVMYIyug|M_;Hc2Zf{w#!;;&Ybaa4-yv?Tv+@2o8{caKcczvr#xe2 zW79G)xbT-5Y%s%iR(W}OR#w)~63HKeqD9TmpPrsx`s#}2)wR*#7cXA?^!|i4sDt#gwzuvz8 zFO1*j1E>@&d3VRscfMWhzJ2?oY$`T5IXi!SGTA@tPzxuhBy@6eGO7BKar13i>B~#5 zldf#fkKdYg)yuh^@9Lh)&r>E&T&NkmOvBD@o@f}@_ye=TZk@XJ-th3|(mQNEoE!H2 z`<2ZrXA^P1_WRtd>+9Ce|NAEW*5>s7tn2G^J)<@xG}hGAh)7GX-m}N%=#e8CCnu@) zUH|>-SJvBGTcr$>THciH2Gz|wc37O9V|lsf6QmRkX!POSaIgOVU+Hb^tgM=bhJl@( zoqE4rTwOEo?kat!#|w4}gIKs?HKgsyzyL3180gzose{!)3^qAS*XI0{Gq#C7?wiC4 z>r0(|%C+XuA*Zzw8=X8oJx`rF6%-gaG5Gg1-DsOQ8HhI?7#)zZE?X0}CSqaT-(M?z zXPa%hd1}WM9*As#@^r>)zu)g)AG$j1rTG3ItuwzuBpCc+-ZhE}2rN+Zoi!s8T0I9e zoo2{>bfojwWBLCYhc`TX_Zgyih0L5hMH!2N1;LM}=|+R9!d0cOuYrnytluvd_uItF zLey#5E@#j-F}bp`@IV9OGBIc>{iDB}A^Yd2r=p?YG!R_P8=JapAS~@#}wmeLXuE=BB+ujA8A3vMHNyrfk0HwKsHw z*Ho>jEg2Ivk3y0QgDTUSEn7?^+ZG<{mCnAuuXa`J?rn>86%-swUtO8Fe*OBDu8^92 z!6z-j1XovA&q-ZfT|WA3qQ|EtXb0OTS3Ui+Y~qET3wMQ;H0LK=JiFICweo)cH{0#E z)@PMcsXzBBdPYUIRpot)zILi8 zPka7;o4oJeFRsbIwc027tI4rfXU^QOxBqr_>h_@i8%Ky`@nqb<(E!zBDc9&zr*JrcyvyOBK zg1Qe%N=hl4|K4xecr%A3!a?=<$^Q$tOC5Xlrc846)rzd8>I?^ZwJx8M+rfU2`+=Dr z^P1`Lb&;FX&R+U_-abFgwxIFdaq8lvAlNb zi=Efh=d(?;FVEQ+Wh?i@=)f<)Y-3v;O?}xJ(q1)EhvGw`N>a0`*Z2 zxA7LsyuQDGKd7H5X`BXYp&by>V_x&+OG)Y5TU(EOTo=1r%hvYpVpm2D+3k7YB&wx6 zoiS{(n(vmZt6G2GmG3XM`gf3BensZxWpesF+uP3efK&W|w}(<2K((V|Gh62IKG{Co z4P|d{EnTuiWo^{fQ(Kt8iWmZR?&R5!l$^Y^{C%8m#0H1#YilN|Y|XhD^!wY};NyL= zXIJrqyqZv&`tzI3K1==sx&@L6o?}^*eA6WFj>Ykf7%i!(GiSd1vdcLA+=}@9bzFRWVRNlYL7^PGzwYm} zEQs?Ow(g8+1T`~u?66q6Y}qM;Isc#EPusk5!-fyb^nUz&KL53U{jbTlwq`#ssUfqC z^=9@eXrjILyomLhu)od1w6n7^zrVX{Rr_nprAwD&`Y$i{zklVZUi`i_pjP!+^ZQq1 z%kMn2sQdWHRX2K@&*$go!_UvN6_u4;`}Fkm?C}`h3n>R8;iommQKHKYo05 zef|8Z@9*B0*r=+yva+)J9gp6Uv9OI-TFcThauTRTye;>(mW|Dwi|T4>ZkCpoR#jhK z_+EvU$*+qmxf9pN?fvp}dOWCIpSSY@sFu**_rqy^{lCi2Z9P3atSl@VIyz@wR#a5% zFfuZdsjZWU6WuPpeX5$izJFz9W#4m4OUvy>kX9N)2*Y&7Yti|8m+shMA!}LWl6z~* z#QppC2UmmofHf00r=QPyb7SMJEt!*>nws+V-71=8oZcrSB;*5aB{AGNpV}~W&YU%| zyUSi)_P5u)7~VPEa_?hW$kJrK)n+iQ14jLxy_>P8>Ef(*z|b6{Pmf}=_%W9t9q90j&0|c4+{&Mw)p%q z-`OHULW{!IMoq1{d*FaWYfFoTEht+sFg%EjS$E*qyWQ_i>i+x?`t5hO=hfBKt9@si z`5gx}Yi5~dTlLxhd@^~KL1I(Zl@$voOc3amvyCd-9UHr=WTMK~x3|OBMs2-xVWBgq z-=CV2l5uK^W>FMM6YHms z9|Lo8)_nT(Nkmw9v8wkpi(9p|wOh;HM)gVJA zv~|$-#rvYe`HE}8*3R0Ne}CQE+uNRfeY-vze+d1hGZ>< zIXm{WZb&~r4^+L(+zM5!`eSF-j{pCDe?7#l|Kh{L!(DxSVOLj$&b{^f`}_AFA3;@3 zXS{Y|qOz*z=jZ43FQ#q284?lkV&^J|WW)dSsSTkaAt`B)Dv-fpWz4(Aug*Jn?wq#{ zn)NiMo@Q9RVMBoa-Y=70U0=T*H2C%9<>g6BRt7Hzb=j7QLdKmMPMu9{FwMTE18V(! zes&f#tns*Fwps3%Yti{LgAmQXs^@ddzZ{d!&p0tb@zd9@r84>V_wD`nb$vauqF?Xp z|F2f_oi)WGE+k}0Qxj9a7_@K`DV9tCHQMLbev9mUT>ty_{VN-j-4W8Ci?15w-ZD9Q z^ytg||NpM<>gxJ((Ov%KkB^T*7H=tg8)cjgDFYl-pB1rQYv#9G;cxfTWwv=fDD+Ho zZ<#Dzw(Qkpf4ji>cC}~LKZ~Hyt@-pAFtDp+(j>+wlo4y*`M6)q4 zd}xqnif{(s%lWy`#N)y1!Q@gk#N9LdqOf4^P_^+`aL*M3u&?^=jqvkbr;%SFVK2wXctRetv#B&sz^QcIu?l46nCdk2Br~ZP+g;(PLh-V~2(D zYlyuJUQdhop~J`rxM0IXSm#!vW-_mtGe>4^(D_72219eluBclVLft4*4ACo*U zZOy)ZWo59s?WYsUS=ZOiwfpnI`7#$Ynx8&1Isht&-|c)Z=XvS4e7(+LgDG?8u5INO zfAw^FyxCd@h?_$4^qALli|em)>y?_cC}+Gb{92M-Q4vqne` z%vYRaS-cE1GQ!Br25NBjJ+`g>cIDaG*}j^55aS9Y6I4}HUVM6bTD32JZ&fHKC#Re) zXcCZtfkES$(E%eKXS3OBk9^eM|L4-jx6qNXhUHUlFu(4zemCK4+T^oopg#T_Cs1E5 zc3(~8;dXxWtsW55L*i7K)?AOP_H9di(WyQ!z{#n}$=O*pmywYXRN<#?zwN6l1W`IQ zypus&PtR}Cl@)=?rEhPA{`>cD86rpAV%AnyKmM`!c%N+P+gqkjpFElJOQV0O_w-e< zyUVyZIRiaBI^LA+w)^`f*u6S0Z{4HEk3%=7`G&3z^WC^{8{QHI^QiE!VKMjV*n3B`_i)V&&`|;`#65A|fK5b(>^da43F$F7)y;-&BdVdv(8a zZGSvywkmkg5VbvT?Z?N*U(f&lXL;EAxV0B91oTRodNH%}U0D~qTcvjY_ptqSe?Y;9u?xA9Kizkh$)`FXY7`Ordp%^A%Dw>G7AKbB_Oo_F`vwr-mb2blfNr=Oj* zbgB1r(CBQ|g$0e%bfcF|)ec{ketzDhB~MRJU!8h-8mQT|tK{XPUTO1JH#R2U+Ecmt z(BZ>Z7do@={c_2>>fg`jrpd=Z4W1oqAp;c*5#7NJSNB$b52`O)e&~?X+PJ+@;p<{1 z-pvd5^6KK^;<~cCJl}o3m~PaH#KUb~etu!o^yqbl%d;LHY6aP*7qeqQE4MgkLh;tdWcQu=SN2pE%i2@~RDXYWwL?%@ zL|lA%^6|dAK_BhYK0P_PtK#FL$A#bCME1vloBRw6@17U2hSk*UVdIrrvebL}sq5Fn zFMZ$szgNaG=;OzaRy98gcIt^h#uhv#En2+zE2ur*U;k$@NW-5$HL2<8)@u))I&&t( z#id0>Q%T9`_qVsjR`DOVi->*K^i4=uz{JdaI#eC%g`-co)~w(6%S%|@ucRe>(v#P( zS91!hy;!||->#?J;-aFO`uhGz!(DF)4tK4NUI(p5u01Vcz2;x{NqFh9WvePbKl3{t z92|UgU+wRr*c6c0|Nr}abwi?a?5+||?eKM1w&mX5l6aV{wXN;dN%i?xS~!J67Q6Ab zwzOn?d2w;m?YFNkE$voWnSXy@*40&0+xTQbMrtZ6I|ncG39PBH(Ov`bQC_hh3x3l{>mW?%P{ zHp>Z^Z&$10$t|Wcp})UBDLEOGs%FfP(2d-5}=@9pNM zrYSRKtT=MSWqK`WxIFLfE~9v1h|9OLFK5u!)m^*j6@<^=^;85hCc?$V_2tdx^I5O1 ztjxNyBCu5iQe-fA=n-KvmOams(U){FM zcXrmZGc#{(&7Qt!@#4^}S)mwX16y-$8nw2zYO1QbYHDiw<(kdzh0WbAE09e1*b6O# zH!w}FPWU?^QNoDl@MXBwcMJ|xJ)c{CWm|4^=*l41&{ZLUyUX5wnZEze(KWZAzPi3$ z;M{!s_~7M!S62iszOpvj-1E}8xz?sNKMHDpzuhinU$-X{J}OisnV=iB#lw5L-qj6> zhh>ic`u=|XwYAaN$9g28lgtbTt*4_oE?>TUb!+x@q2s&D-md!cr3BH$Dwa$z&AjBY zI&7_0;h$!HyMW8fd~Fdeil@3QoSd9eCK(gT-rRV2gBdgn?B(U9WqJeDXkuVsumKIK z+_(|(_U`WRtyxzu%`(m2l6F?=>Z(xh-{0O|ZeV0aYj%mq%C0THUweH`6vV>~LNV*0 z6Zs5l8eoBmw1A_SYXfLzdC{UpU9e8E#zD;kuWoEyyzlR~XwRU>$9lisEx#Y?0Uw=P z&Y-QMZnyuWDU&8G z+L(M?2s)&@z-cPShA!RseRn>7z8+t%>j|nFo@+zgz)*MKc&Kw=;6zY=;lhOgS65d} zEiJFa#6-}f{f#@&#{06uolLKfipO8slzQ5v{9R1x>ubJ~wiG}2J3GS=)TEzfnmr9A z8Gu#=RJ~ri{mTCO`r{kR-ribS_xG2SWswT3@8a|Dlh~op&(E)R?G{t*+n#aJ>G}Ej z-!tH0Hk~o-_qVrOD?UCFIxemsx8~d1+tph%AYNTC<8(B~^Ru&;*ZuvK`T5z|O}F1p zTJq#c%3Ry(u-Dht8Y4Ao!V^G&tL``FLIWc+sAcx%O%AV&MF6OeYWw|8a#}CM$Q4Io z)*YAxT8y$W(i$S9HRUvH2EIX502-Lci*=T=MYJWpI5X4u)Y-G9u)h0&&eII5JtwPK zRexJ^6F$Q1#k--Vrsm3)%wW(wY}$2b2gssuC;PQKJBv*-FR92{79BYQ@2u6WlWOCY zUItp%QTzKF7Y9ecRISiCx6aSEN9lfLuiu-N1YIoS@G^!iqOh-FE;n~wyXE{R(}QcXg)nXJ!y&l{y#yN zm;0L|g(mYF@wf^{{r!J7ndRPE($K(g{Np^^YEX0I>8=xVEQ?FN-z{HV_4U=N;N^XC zRWB4}?d$HW^vBx3dR6@FjO5qX*Fp1HAmwVlvzENRzJB^@=zxEP_;ki=4-dDSCLCxG zI{x~~%EkHn|89#Bzqzk=_oK&;x2B($%e}S5la-bA)$Q%;*Tw97^uoCAPetvglj>1> ztG0rAF2&E!Wj;92*!S(wAt#VcKY#wbwYPdZs4)iWNM2s%s~xTjiR%w~dgg`v+}y!c zRlB52v!0avInc?6PQ6Waf2)>1*A80~FxRRSl(pvn|MMKw5CFAQ$ZK6K4^4tt zyQ6kTOyk!Vi~C>wd_F(i%?&iE+SS#yCHuPG-*4OZg9_2Smy5*p{M=+!_GZO0-`V#ru|9tMxb(>h!MAsIex7zNSZuPo|FsrQVVU#AMMWGNA%^Vf zInA&-;&2)d*$Bqt;r_Y~jTU+1G6b(yDTc)I> zBvbqC?d|6!`ySb(2BAGEATLZZUk6q4B%B<rQT!1D9*`ZNw&k{*n?Xs*KVH1Z`1$*{wx;IE!Z(E5QAO7wT|$QU4~kf?ZOOcB zQva`}SI+j9ppvS(diI|mA3^=Ev~zPJpLHKOemwi>sj0ty-~Yev)z#J3-_~&P@`g^; z3Kfx(TBSa}CP-M#2h=GL5fXZ|BX7E1tcqvt?{A^2!`7OVya)g-QjwIDoMn{S_2}{A z$W%!Ctw6e1G66JX4w~)2ZeQqpmNIg+n{#G(W6JFu7>sjE99p$hB-PmT9v*Ec~ntSvSnTT{{Gc2=XSo*HFJI~1XX6vZ9E?l<^6Ke2w^p!f}%HT;$qPAwO3|kvz zRrO_s=Tk`Ww_sB-*M@C*ccTJJAr4?zTC|fLTDl~#>g{heFgFkH7Sp|SZEf__sZ*CO zTBNl6oBqBZhaSVG!}OTffMP@1Jnzc-y06-Qzbv<(dN$2?vmV4ihfBOnYqn-z&$_#- z^wqVsvr#4#L-*I!hOUphdj`?2V2h~#aajJ#pU>xC+yDRBkEHyb(~XtE>QA3NTUGe@ z7#APkG|+JI@1LKa+ak|x+Ei@#_xt|;tZ#2_a`E%0Ll@*Rgm6q}y!P(yZqSJFx3{-v zM?$+M4~{&Qi_neQa$#%s^)rY`r_=`1@^>-5b1VXHZ_B-0e!q74oH;Vfp(7s&Y}2<3 z+$+CdJ84PW-YQF3b5Mugx3#r(-dgBr%LQJhHEW}{&q5lRWDNWM?(W$&kZi^P8q~`^ z-Y08b^YDEAzvAuUQ1L(d(;2Vr{eCa{(TuZc#edGoftKPiFfbT2pUzj@Q}D3qva+_e z_v`EHuiq`dzqVJ}d=|=3l#{cwx6JFy%lny_m`oB6u`FG(q+*K!G$M8#;YgVk#KpJ#1IxuO; zk|ipts;ZjW+TO0Nu70_9+#vm12ICz(8-#>}Op2fRl)k#+32E%1CZy}}^|9JvYc5<^ z=zJD25cPPKqP=~5?Cvtt&CsN~fMq)4wP1hSrBk)TXCX~jM{^Vx7lY;sGd@2%3mT~U z^Y`!EThGtWM`YOpP!$brkoIHZE4Za(ed%O`?*9feqRpK;v$xa@O3ePzrVemI%7se*6%%)pSKh| zblUsx*K5$qP|%dZj2RM7pFOks#Up9t0$SL%%y%|u0Qc>Ujn3z7KKDpIg(TYrOh0E9 zPW=6TfBg6R_4O-r+Y(=Fy&iXYie~Va2hIFmlkQbMmu+ot&ptcLwCc-6_o=gHt@`@< z`s>ft+wt1Eh{U2 ze$LC!Z<>m#y1F)KF;ARl(EEFPpFV!PcyIOhS#idY$T^dIiffJZ$rC3gsI<1XTg&qE z^OtW)O-WhdH`mJVZCza*Xg$he|IAOHK24cE{dvW_$&(+i+XN{U6DD!RH$L?QEl$eF zSn*g;QZn+QDnx3*94(szT^$`0R8rH@tYpuhIa9L5)x||aTl@6NzrVh|wyOWPXR&{# zn*Th|NT!us`uTaK#n2Y+hb>(E8>CFLmUMJ*sH&=3_C0;_B;)@C-n_|K zx@3vPubi8kTwPsVU)|Zc`E)Tm8=IDj%8{24>lxlBosx@~t{;C5Gy+gDZ`!nF&FuW3 z0aMVDDbRXM(A>N4Y%|advPtQykgKc1{rPRbOi)>wdV1Qf_xpZ-`EuF+_1Ek1;f{`t zZoN`V-Q{biPvyme1pU$6-_-oKTw7%K5x5RJAB=Xnd$Qa z%gVOZecL=g^Y5>(TwGj5d!B(7@PW*V+go+=_;L4*8#hMnC|D>7PaQLb67E*N-)qEk zICb~gw998^8k-hB^O;@?nxvYl9bUD_1`=lx5v}hVAvHC_9S&G02k22jfBAe1q z?n*S?E&!>DoRbIQop6t47uwH~A0k%GA4Yr_9hD<{o$UR*4 z>Gt=qt3oV?&Cbhpx zYCj$o&$_>FFQ`pFb!zCvrH~Lm;Jiocz%2Xvee>#mW&V9;zCRE&TKentdTk4fi28q@ z=WFjW0XvqVqT~9T#;Nn?ub(nS#CM*}&B8OskGt=!{vK9TRJ3ya$#xPi*-ow(2NX@v+(Vhv)o?*A0ID z|MyOtzo#!i)kc=!H(^XTfz-_aRs!oT1&9$_S?S4=WV~w z0d@QKR)3#XakfX&c*=wc328GS;h@pHIsN=9$7Z(FSFcUAH{>0IPl&&HlLMMPIWyCk ziJU{d}JHK6`0#^xFMXvre8o8JLo?1hlSt!UTcs^Y?Gs(?9QB{?e#D zmFFsob;DlAM?UMBdav03`}QE+Gm`IL*6_lVX61@ko9qUjppCaW?B=qHS6T{>)|tJ&h$HO44%Iae3cF9yhJ>z$jHc;Ss(oK z*RQH2>!+T}-TE(V)zSzL502tvhyJb&@ixq#V|*{B`|7XoX_m=Lzu&nG7JSyT^>_5j zX-Q`!vq5e;x~CA+p_5MS*|1{9^_)j?0s;$OyvP6-+oGbPnwpv?Uruu>@9yva?h&s& z_jcCbwIQd^Za5XO_M2VUvW?TGZ|}S6wKZaGed`y~DG*=W5pb-lBk;Bk`@b*!zuwAT4_Z9>>-zqG zOFw-0AYWVS=hx@t>|FZs(a|FHlPl(6-I#N#+mhM1;;e4VB$JE_3bmiln$I%Jon>TX z1Zw20jo-h|=i$^Ze|1|gFZb8~y3J{U$r;Hh;>B!FkIr$dG)xbOgL4}t_hiBHvX=b% zo3O^uRAIbj_2*R#+2`h1?)v?1cj(@#(yJ?j-9c4SpS=CME9&Ov;U7PKTot)_*{aah z%Wm9=0M)BsC5_X17A;;ZDj~6gPu6Nm)mmO@GtfHZRdIWFWnEvl_Qs8fzrXMQ|90WT z1I@H^GecL0t#oeZ1FiiD37HaC`Bc=Oms{>hpXKML)8n%~JU9s2wK7#J^vb4G@4p}W z>vam(=HA}+Qr-S%ClfRC)g_+7Ya=!ux_r2~eTHfNrOTJU-bn5TZNm8U@#DoEg^$lj z$k`lNWBK`_yF6&oEoi}?lao`*_S>Mf_rb?Fz8hxmdvj-JFlapB>?~8zGXAX@7oE7p zbPQ5^3iOTt+Xd%bUl;r7^XKB9<*5<#BR{9z-j=)RW{#?yq@?7s9^reZS2L}7dwaXK zon2h@w>KA;`OXGSKJTmj{p!?I?LNE5$NR6V&#zf@b#-|5?{9Cd%HPdl&z_U`E|cmw(&?VdU&|~^@)kf zQJYdaL7P#Ic8gc%MKQnX66cJFwyx;eGU7wuoD)IQ5z{%?VWwZD*Gc!RGuvK4P zENo`y&$_dt5HvUbqkZXgqwIT6o<3b!_V(5+qtvcj+w=2ZNj!{8J3CX>x@?92e7mUa zd2@q;g03uf=ePZEfEm;sUHSOvTEpypA#0;R4U|iLver{(&03`yybQDyrRs67xsR-V z-rUIIWf5g%+dyM`aeJ#~8rfB}-TeIQ$H&KB{{H1Qudb|2K4tOnD3AWMsZ&!ujJD<9 z&y#3NOi53d{`c?Su~%1D%hm=32lINm6%`eAUiY7GcQ@nyzS_>mtHalu{rU5!=g=Xi z;~VS$JeF@en`Zp!^XI$v0yB;U7pHxAaPSzY3%fj-TU^gX$|NH}L{xNVxq!jdwdZCg zDZBTXRDF5zu;SmH;`5Sgqqmp6GS0Z5psc2L?5whb>xV0QelC1>ceix%@xIdRyDKcx z&df|&8@1I)NJ!{Tu!UC3%+IL@8W@kQum79jP_T*DjH1+uX z)YNRL`B@|?Cid#l(e7WTZh?Z`eCv!~)>A)lw0pI@0(yUNicN2X-oJGjfnG{4}tQT90zQPG!|mU^Gj z)=$4yV_EQ^q3r#=weRlk-o3IYM8yBx%%`WPzy9*_@+`aBUF=%!uAjc7`?mARhHTG^ z{rd8<`|h&0Ma5rt$X4cpmbok`e0*$W^!7Z^fRc))va&NHBO@0#ckwG}Ik~uXzcZyx zvzCCST+8?Wo(n45#l)7a-~TUab=caX=K={v{^w@Sm?2U7<)XWnmltSdv)>$xg`iDO zllI-;%y#c&Y;oH6cXxk1Xy*TN#`wHNUusHMe^hifS+I)Y|VZl)jUo@p3w_xp`8XdwZpY~j}516stL@!`S2 z?LlT*6P7>oNlaV_n$%NOQCShWIqjX6xY)y!=Vv`TI~%;F<$3Jxva2hDmv2cp$n^KA ze*Mm+y9%p6pJK?qvLY~QciGyqH#Zit@k&|rxx2W)7TK*VfQ&Yts`+>1h)Zi*Th_fj zm3_&e(nU#G8MH>U^yQ_ax{n(wZEVwX%x2g9|9wCE+nbxGPMipMdu!{|DN|HDv+nHJ zc>5UR=N-xCWUR}4_EvqpWPILcv1afxi>nJNJALMi4`}zK*Ho={@&Y~&Usg8X z?vE^f_V@e!>!8m6)z#tCchpMWJG|!Sr4@n7oPVr?9x+xdg@uK)OtZr}U*C9J zw!GNm(lX!Kd0F=sKAH*2lm^A;_I$rr{p)Og-Qw@}s{Q}|`uh1y!-GK4pOV2LAxrk` zvH8x!vdl2~TuVy}XjvMlI(TeY^P|8t|6U9`zg)=kb8~I2J$A&p)?JgasR$^2eXaDo zL&5J=8{R(7Khzz!ek+_PYTLh}22dzbZz zZ$Bqn{OrTS!)|YHZS9T!|LeNn^An)0D3-Oqwg}3|8_8f7Eq3ppWfr!hJ+c_wVSV`gdAf?4n$_2n=gygx z?%83Qc6R#FZgJ`VKaSh?2&?-^sQb+^xL5UBcX8F%SI4^b_hr1QD13e9Pw~t4A7W1L z)2{7_+>p??G3~6><0mI4AA5MXy?0~saVa}HyJMi$K(hLK`omcxCVAEB#P7Rv@%gQ- z*~d;yRPMEYx8pDeAD}8G(duB&|e)fI; z|GkCRW6LES9Ua|zrA&LO-|xMCV^eB(r*iA|O_FVie{N_&XNwjrs<;XpzFbj*cZC_Y d!vMbhXLxyk&dbT$EuMioC!Vf;F6*2UngB!&OrZb( literal 0 HcmV?d00001 diff --git a/doc/source/figs/NO2/Copernicus_S5p_NO2.png b/doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-versions.png similarity index 100% rename from doc/source/figs/NO2/Copernicus_S5p_NO2.png rename to doc/source/figs/NO2/Copernicus_S5p_NO2_inquire-versions.png diff --git a/doc/source/s5p-no2.rst b/doc/source/s5p-no2.rst index 55b24fc..c465e16 100644 --- a/doc/source/s5p-no2.rst +++ b/doc/source/s5p-no2.rst @@ -260,7 +260,7 @@ of the *Product Algorithm Laboratory* portal. Also this will produce a table fil To visualize what is available from the various portals, the :py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: -.. figure:: figs/NO2/Copernicus_S5p_NO2.png +.. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-versions.png :scale: 50 % :align: center :alt: Overview of available NO\ :sub:`2` processings. diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index b389fa9..665ee81 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -306,7 +306,7 @@ should have created an overview figure next to the table file:: The figure should look like: -.. figure:: figs/NO2/Copernicus_S5p_NO2.png +.. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-versions.png :scale: 50 % :align: center :alt: Overview of available NO2 processings on DataSpace. diff --git a/src/cso/cso_colhub.py b/src/cso/cso_colhub.py index 539225d..89decab 100644 --- a/src/cso/cso_colhub.py +++ b/src/cso/cso_colhub.py @@ -302,7 +302,7 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): # endfor # archive_dir # info ... - logging.warning(f"{indent}save ...") + logging.info(f"{indent}save ...") # sort: listing.Sort(by="orbit") # save: diff --git a/src/cso/cso_file.py b/src/cso/cso_file.py index 0ad1114..1cb4e51 100644 --- a/src/cso/cso_file.py +++ b/src/cso/cso_file.py @@ -1688,8 +1688,8 @@ class CSO_Listing(object): for key in self.df.keys(): selection = selection.replace(f"%{{{key}}}", f"df['{key}']").strip() # endfor - # testing ... - logging.info(f"{indent}selection `{selection}` ...") + ## testing ... + #logging.info(f"{indent}selection `{selection}` ...") # adhoc: replace some operators that are not allowded anymore ... selection = selection.replace( " and ", " & " ) # evaluate: diff --git a/src/cso/cso_inquire.py b/src/cso/cso_inquire.py index 0944609..772271e 100644 --- a/src/cso/cso_inquire.py +++ b/src/cso/cso_inquire.py @@ -34,6 +34,9 @@ # 2026-03, Arjo Segers # Fixed sorting of collections and processors for latests versions of pandas. # +# 2026-03, Arjo Segers +# Added `CSO_Inquire_Plot_Avail` class. +# ######################################################################## ### @@ -59,7 +62,8 @@ The classes and are defined according to the following hierchy: * :py:class:`.UtopyaRc` - * :py:class:`.CSO_Inquire_Plot` + * :py:class:`.CSO_Inquire_Plot_Versions` + * :py:class:`.CSO_Inquire_Plot_Avail` @@ -85,12 +89,12 @@ import utopya ######################################################################## ### -### modules +### plot versions ### ######################################################################## -class CSO_Inquire_Plot(utopya.UtopyaRc): +class CSO_Inquire_Plot_Versions(utopya.UtopyaRc): """ Create plot of data version versus time to indicate the available orbits. @@ -120,10 +124,10 @@ class CSO_Inquire_Plot(utopya.UtopyaRc): The created plot shows a time line and on the vertical ax the processor versions; a bar indicates when a certain version was used to process orbits: - .. figure:: figs/NO2/Copernicus_S5p_NO2.png + .. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-versions.png :scale: 50 % :align: center - :alt: Overview of available NO2 processings. + :alt: Overview of available NO2 processing versions. Specify the name of the target plot:: @@ -164,7 +168,7 @@ class CSO_Inquire_Plot(utopya.UtopyaRc): # info ... logging.info(indent + "") - logging.info(indent + "** create inquire plot") + logging.info(indent + "** create inquire plot of versions") logging.info(indent + "") # init base object: @@ -484,13 +488,260 @@ class CSO_Inquire_Plot(utopya.UtopyaRc): # info ... logging.info(indent + "") - logging.info(indent + "** end inquire plot") + logging.info(indent + "** end inquire plot versions") + logging.info(indent + "") + + # enddef __init__ + + +# endclass CSO_Inquire_Plot_Versions + + +######################################################################## +### +### plot availability +### +######################################################################## + + +class CSO_Inquire_Plot_Avail(utopya.UtopyaRc): + + """ + Create plot with bargraph of number of orbits versus time to check availability. + + The information on orbits is taken from a csv table created by for example + the :py:class:`CSO_DataSpace_Inquire` class. + Specifify the name of the table file in the settings:: + + ! listing file: + .file : ${my.work}/Copernicus/Copernicus_S5p_NO2_%Y-%m-%d.csv + + The date templates are by default filled for the current day. + Alternatively, specify an explicit date:: + + !~ specify dates ("yyyy-mm-dd") to use historic table: + .filedate : 2022-01-28 + + The plot could also be created by combining multiple tables; + use a semi-colon to seperate the file names (and eventually the dates):: + + ! listing files: + .file : ${my.work}/Copernicus/Copernicus_S5p_NO2_%Y-%m-%d.csv ; \\ + ${my.work}/Copernicus/Copernicus_S5p_NO2_pal_%Y-%m-%d.csv + !~ specify dates ("yyyy-mm-dd") to use historic tables: + !.filedate : 2022-01-28 ; 2022-01-28 + + The created plot shows a bar graph with one or more time series with the number of orbits per month: + + .. figure:: figs/NO2/Copernicus_S5p_NO2_inquire-avail.png + :scale: 50 % + :align: center + :alt: Overview of number of available NO2 oribits. + + The time series are defined by keywords:: + + .series : RPRO OFFL + + For each keyword, define the label (used in the legend), a selection filter, and the bar style:: + + .serie.RPRO.label : RPRO + .serie.RPRO.selection : (%{collection} == '03') & (%{processing} == 'RPRO') + .serie.RPRO.style : color="green" + + Specify the name of the target plot:: + + ! output figure, date of today: + .output.file : ${my.work}/Copernicus_S5P_NO2_%Y-%m-%d.png + + Optionally define a creation mode for the (parent) directories:: + + ! directory creation mode: + .dmode : 0o775 + + The following flag is used to ensure that the plot is renewed:: + + ! renew existing plots? + .renew : True + + """ + + def __init__(self, rcfile, rcbase="", env={}, indent=""): + """ + Convert data. + """ + + # modules: + import os + import numpy + import datetime + import pandas + import matplotlib + import matplotlib.pyplot as plt + import matplotlib.dates as mdates + + # tools: + from . import cso_file + + # setup graphics back-end: + matplotlib.use("Agg") + + # info ... + logging.info(indent + "") + logging.info(indent + "** create plot with data availablity") + logging.info(indent + "") + + # init base object: + utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) + + # directory creation mode: + dmode = self.GetSetting("dmode", totype="int", default=None) + + # renew output? + renew = self.GetSetting("renew", totype="bool") + + # table file(s) to be used: + filename_templates = self.GetSetting("file").split(";") + # count: + nfile = len(filename_templates) + # time stamp in file? + filedates = self.GetSetting("filedate", default=(";" * (nfile - 1))).split(";") + + # list with time expanded: + filenames = [] + for ifile in range(len(filename_templates)): + # current: + filename_template = filename_templates[ifile] + # time? + if len(filedates[ifile]) > 0: + t0 = datetime.datetime.strptime(filedates[ifile].strip(), "%Y-%m-%d") + else: + t0 = datetime.datetime.now() + # endif + # expand time templates: + filenames.append(t0.strftime(filename_template.strip())) + # endfor + + # current time: + t = datetime.datetime.now() + + # annotation: + title = t.strftime(self.GetSetting("title")) + + # target file template: + figfile_template = self.GetSetting("output.file") + # target file: + fig_file = t.strftime(figfile_template) + + # create? + if (not os.path.isfile(fig_file)) or renew: + # info .. + logging.info(f"{indent}create {fig_file} ...") + + # not yet for multiple .. + if len(filenames) > 1: + logging.error(f"only single inquiry table file supported yet ..") + raise Exception + #endif + # read: + lst = cso_file.CSO_Listing( filenames[0] ) + + # time range: + t1 = lst.df["start_time"].min() + t2 = lst.df["start_time"].max() + # reset to start/end of year: + t1 = pandas.Timestamp(f"{t1.year}-01-01 00:00") + if not t2.is_year_end: + t2 = pandas.Timestamp(f"{t2.year+1}-01-01 00:00") + #endif + + # collection frequency: + tfreq = "MS" # start of month + tt = pandas.date_range(f"{t1.year}-01",f"{t2.year}-12",freq=tfreq) + # storage at full freqency: + ndf = pandas.DataFrame( index=tt ) + + # keywords for plots: + series = self.GetSetting( "series" ).split() + # loop: + for serie in series: + # info .. + logging.info(f"collect serie `{serie}` ...") + + # settings: + selection = self.GetSetting( f"serie.{serie}.selection" ) + # info .. + logging.info(f" selection: {selection} ...") + + # select records: + xlst = lst.Select( expr=selection, verbose=False ) + # check ... + if len(xlst) == 0: + logger.error(f"no records found matching selection `{selection}`") + raise Exception + #endif + + # group on start time, per start of month: + grp = xlst.df.set_index("start_time").groupby( pandas.Grouper(freq="1MS") ) + # series with months as index and number of grouped records as values: + nn = grp.size() + + # store at full frequency: + ndf.at[ nn.index, serie ] = nn.values + #endfor + + # new: + fig = plt.figure(figsize=(12, 4)) + ax = fig.add_axes([0.05, 0.07, 0.94, 0.85]) + # no bottom yet: + bottom = None + # loop: + for serie in series: + # current values: + nn = ndf[serie].values + + # plot features: + label = self.GetSetting( f"serie.{serie}.label" ) + style = self.GetSetting( f"serie.{serie}.style", totype='dict' ) + # add bar plot, left side of bar at time value (start of month) + ax.bar( ndf.index, nn, width=25, bottom=bottom, align="edge", label=label, **style ) + + # update bottom: + if bottom is None: + bottom = nn + else: + bottom = bottom + nn + #endif + #endfor + # annote: + ax.legend() + ax.set_ylabel(f"number of orbits per month") + # time axis: + ax.set_xlim((t1, t2)) + ax.xaxis.set_minor_locator(mdates.MonthLocator()) + ax.xaxis.set_major_locator(mdates.YearLocator()) + ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y")) + ax.grid(axis="x") + + # directory: + cso_file.CheckDir(fig_file, dmode=dmode) + # save: + fig.savefig(fig_file) + + else: + # info .. + logging.info(indent + "keep %s ..." % fig_file) + + # endif # renew + + # info ... + logging.info(indent + "") + logging.info(indent + "** end inquire plot avail") logging.info(indent + "") # enddef __init__ -# endclass CSO_Inquire_Plot +# endclass CSO_Inquire_Plot_Avail ######################################################################## -- GitLab From 6dfd3be37f352a9e8706b6060603d8518d32ea84 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Tue, 21 Jul 2026 15:30:30 +0200 Subject: [PATCH 3/3] Changed counting of files in S3 bucket following deprecation error. --- src/cso/cso_dataspace.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cso/cso_dataspace.py b/src/cso/cso_dataspace.py index c793802..aff1bd4 100644 --- a/src/cso/cso_dataspace.py +++ b/src/cso/cso_dataspace.py @@ -58,6 +58,9 @@ # Trap files that could not be downloaded. # Added production time of data files to listings. # +# 2026-07, Arjo Segers +# Changed counting of files in S3 bucket following deprecation error. +# ######################################################################## @@ -782,7 +785,9 @@ class CSO_DataSpace_Downloader(object): pfiles = bucket.objects.filter( Prefix=product ) # count: - npfile = len(list(pfiles)) + npfile = 0 + for pfile in pfiles: + npfile += 1 # check ... if npfile != 1: print(f"ERROR - found {npfile} files in S3 bucket for product: {product}") -- GitLab