This exercise builds on output from the parallel processing exercise. It does not address how projections and navigation is dealt with in BALTRAD. This should be addressed in a separate exercise.
The Cartesian product area used in this exercise is pre-configured and looked up from a registry.
Rudimentary composite¶
%matplotlib inline
import glob, time
import matplotlib
import _raveio, _rave
import _pycomposite, compositing
import warnings
warnings.filterwarnings("ignore") # Suppress SyntaxWarning from Python2 codeimport os
import urllib.request
from pathlib import Path
# Set the URL for the cloud
URL = "https://js2.jetstream-cloud.org:8001/"
path = "pythia/radar/erad2024/baltrad/baltrad_short_course/"
!mkdir -p data
files = [
"seang.h5",
"searl.h5",
"sease.h5",
"sehud.h5",
"sekir.h5",
"sekkr.h5",
"selek.h5",
"selul.h5",
"seosu.h5",
"sevar.h5",
"sevil.h5",
]
for file in files:
file0 = os.path.join(path, file)
name = os.path.join("data", Path(file).name)
if not os.path.exists(name):
print(f"downloading, {name}")
urllib.request.urlretrieve(
f"{URL}{file0}", os.path.join("data", Path(file).name)
)downloading, data/seang.h5
downloading, data/searl.h5
downloading, data/sease.h5
downloading, data/sehud.h5
downloading, data/sekir.h5
downloading, data/sekkr.h5
downloading, data/selek.h5
downloading, data/selul.h5
downloading, data/seosu.h5
downloading, data/sevar.h5
downloading, data/sevil.h5
generator = compositing.compositing()
files = glob.glob("data/se*.h5")
print(files)
generator.filenames = files
print(type(generator))['data/searl.h5', 'data/selek.h5', 'data/sekkr.h5', 'data/seang.h5', 'data/seosu.h5', 'data/selul.h5', 'data/sevar.h5', 'data/sevil.h5', 'data/sehud.h5', 'data/sekir.h5', 'data/sease.h5']
<class 'compositing.compositing'>
# Run with all defaults to a pre-configured area that uses the Google Maps projection.
# First two arguments are product date and time. These are taken from the last input file if not specified.
before = time.time()
comp = generator.generate(None, None, area="swegmaps_2000")
print(type(comp))
after = time.time()
rio = _raveio.new()
rio.object = comp
rio.save("data/comp_pcappi1000m.h5")
print("Compositing took %3.2f seconds" % (after - before))--- Logging error ---
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 1017, in emit
self.socket.send(msg)
~~~~~~~~~~~~~~~~^^^^^
OSError: [Errno 9] Bad file descriptor
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 888, in _connect_unixsocket
self.socket.connect(address)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 1020, in emit
self._connect_unixsocket(self.address)
~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 899, in _connect_unixsocket
self.socket.connect(address)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory
Call stack:
File "<frozen runpy>", line 203, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel_launcher.py", line 18, in <module>
app.launch_new_instance()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/traitlets/config/application.py", line 1075, in launch_instance
app.start()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelapp.py", line 758, in start
self.io_loop.start()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/tornado/platform/asyncio.py", line 211, in start
self.asyncio_loop.run_forever()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
self._run_once()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
handle._run()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/events.py", line 89, in _run
self._context.run(self._callback, *self._args)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 701, in shell_main
await self.dispatch_shell(msg, subshell_id=subshell_id)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 469, in dispatch_shell
await result
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/ipkernel.py", line 379, in execute_request
await super().execute_request(stream, ident, parent)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 899, in execute_request
reply_content = await reply_content
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/ipkernel.py", line 471, in do_execute
res = shell.run_cell(
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/zmqshell.py", line 632, in run_cell
return super().run_cell(*args, **kwargs)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3116, in run_cell
result = self._run_cell(
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3171, in _run_cell
result = runner(coro)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/async_helpers.py", line 128, in _pseudo_sync_runner
coro.send(None)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3394, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3639, in run_ast_nodes
if await self.run_code(code, result, async_=asy):
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3699, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "/ipykernel_1527/1133040516.py", line 4, in <module>
comp = generator.generate(None, None, area="swegmaps_2000")
File "/srv/conda/envs/notebook/rave/Lib/compositing.py", line 146, in generate
return self._generate(dd, dt, area)
File "/srv/conda/envs/notebook/rave/Lib/compositing.py", line 285, in _generate
logger.info("Using legacy compositing")
Message: 'Using legacy compositing'
Arguments: ()
<class 'CartesianCore'>
Compositing took 11.19 seconds
Tweak the plotter from earlier exercises¶
# Two color palettes, one used in GoogleMapsPlugin, and the other from RAVE
from GmapColorMap import dbzh as dbzp
# Convert a 768-list palette to a matplotlib colorlist
def make_colorlist(pal):
colorlist = []
for i in range(0, len(pal), 3):
colorlist.append([pal[i] / 255.0, pal[i + 1] / 255.0, pal[i + 2] / 255.0])
return colorlist
# Convert lists to colormaps
dbzcl = make_colorlist(dbzp)
# Then create a simple plotter
import matplotlib.pyplot as plt
StringType = type("")
def plot(data, colorlist=dbzcl, title="Composite"):
mini, maxi = data.shape.index(min(data.shape)), data.shape.index(max(data.shape))
figsize = (20, 16) # if mini == 0 else (12,8)
fig = plt.figure(figsize=figsize)
plt.title(title)
clist = (
colorlist
if type(colorlist) == StringType
else matplotlib.colors.ListedColormap(colorlist)
)
plt.imshow(data, cmap=clist, clim=(0, 255))
plt.colorbar(shrink=float(data.shape[mini]) / data.shape[maxi])plot(
comp.getParameter("DBZH").getData(),
title="Default composite: DBZH 1000 m Pseudo-CAPPI, nearest radar",
)
Maximum reflectivity, lowest pixel, add QC chain¶
generator.product = _rave.Rave_ProductType_MAX
generator.selection_method = _pycomposite.SelectionMethod_HEIGHT
generator.detectors = [
"ropo",
"beamb",
"radvol-att",
"radvol-broad",
"rave-overshooting",
"qi-total",
]
before = time.time()
comp = generator.generate(None, None, area="swegmaps_2000")
after = time.time()
rio.object = comp
rio.save("data/comp_max.h5")
print("Compositing took %3.2f seconds" % (after - before))--- Logging error ---
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 1017, in emit
self.socket.send(msg)
~~~~~~~~~~~~~~~~^^^^^
OSError: [Errno 9] Bad file descriptor
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 888, in _connect_unixsocket
self.socket.connect(address)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 1020, in emit
self._connect_unixsocket(self.address)
~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 899, in _connect_unixsocket
self.socket.connect(address)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory
Call stack:
File "<frozen runpy>", line 203, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel_launcher.py", line 18, in <module>
app.launch_new_instance()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/traitlets/config/application.py", line 1075, in launch_instance
app.start()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelapp.py", line 758, in start
self.io_loop.start()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/tornado/platform/asyncio.py", line 211, in start
self.asyncio_loop.run_forever()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
self._run_once()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
handle._run()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/events.py", line 89, in _run
self._context.run(self._callback, *self._args)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 701, in shell_main
await self.dispatch_shell(msg, subshell_id=subshell_id)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 469, in dispatch_shell
await result
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/ipkernel.py", line 379, in execute_request
await super().execute_request(stream, ident, parent)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 899, in execute_request
reply_content = await reply_content
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/ipkernel.py", line 471, in do_execute
res = shell.run_cell(
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/zmqshell.py", line 632, in run_cell
return super().run_cell(*args, **kwargs)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3116, in run_cell
result = self._run_cell(
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3171, in _run_cell
result = runner(coro)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/async_helpers.py", line 128, in _pseudo_sync_runner
coro.send(None)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3394, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3639, in run_ast_nodes
if await self.run_code(code, result, async_=asy):
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3699, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "/ipykernel_1527/4106185503.py", line 12, in <module>
comp = generator.generate(None, None, area="swegmaps_2000")
File "/srv/conda/envs/notebook/rave/Lib/compositing.py", line 146, in generate
return self._generate(dd, dt, area)
File "/srv/conda/envs/notebook/rave/Lib/compositing.py", line 285, in _generate
logger.info("Using legacy compositing")
Message: 'Using legacy compositing'
Arguments: ()
Compositing took 68.21 seconds
plot(comp.getParameter("DBZH").getData(), title="Maximum reflectivity, lowest pixel")
Plot correspondong total quality index¶
dbzh = comp.getParameter("DBZH")
qitot = dbzh.getQualityFieldByHowTask("pl.imgw.quality.qi_total")
plot(qitot.getData(), "binary", "Total quality index")
Now use “total quality” as the compositing criterion¶
generator.qitotal_field = "pl.imgw.quality.qi_total"
before = time.time()
comp = generator.generate(None, None, area="swegmaps_2000")
after = time.time()
rio.object = comp
rio.save("data/comp_qitotal.h5")
print("Compositing took %3.2f seconds" % (after - before))--- Logging error ---
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 1017, in emit
self.socket.send(msg)
~~~~~~~~~~~~~~~~^^^^^
OSError: [Errno 9] Bad file descriptor
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 888, in _connect_unixsocket
self.socket.connect(address)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 1020, in emit
self._connect_unixsocket(self.address)
~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
File "/srv/conda/envs/notebook/lib/python3.13/logging/handlers.py", line 899, in _connect_unixsocket
self.socket.connect(address)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory
Call stack:
File "<frozen runpy>", line 203, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel_launcher.py", line 18, in <module>
app.launch_new_instance()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/traitlets/config/application.py", line 1075, in launch_instance
app.start()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelapp.py", line 758, in start
self.io_loop.start()
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/tornado/platform/asyncio.py", line 211, in start
self.asyncio_loop.run_forever()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
self._run_once()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
handle._run()
File "/srv/conda/envs/notebook/lib/python3.13/asyncio/events.py", line 89, in _run
self._context.run(self._callback, *self._args)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 701, in shell_main
await self.dispatch_shell(msg, subshell_id=subshell_id)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 469, in dispatch_shell
await result
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/ipkernel.py", line 379, in execute_request
await super().execute_request(stream, ident, parent)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/kernelbase.py", line 899, in execute_request
reply_content = await reply_content
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/ipkernel.py", line 471, in do_execute
res = shell.run_cell(
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/ipykernel/zmqshell.py", line 632, in run_cell
return super().run_cell(*args, **kwargs)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3116, in run_cell
result = self._run_cell(
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3171, in _run_cell
result = runner(coro)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/async_helpers.py", line 128, in _pseudo_sync_runner
coro.send(None)
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3394, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3639, in run_ast_nodes
if await self.run_code(code, result, async_=asy):
File "/srv/conda/envs/notebook/lib/python3.13/site-packages/IPython/core/interactiveshell.py", line 3699, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "/ipykernel_1527/3132186426.py", line 3, in <module>
comp = generator.generate(None, None, area="swegmaps_2000")
File "/srv/conda/envs/notebook/rave/Lib/compositing.py", line 146, in generate
return self._generate(dd, dt, area)
File "/srv/conda/envs/notebook/rave/Lib/compositing.py", line 285, in _generate
logger.info("Using legacy compositing")
Message: 'Using legacy compositing'
Arguments: ()
Compositing took 21.59 seconds
plot(comp.getParameter("DBZH").getData(), title="Maximum reflectivity, quality-based")
plot(
comp.getParameter("DBZH")
.getQualityFieldByHowTask("pl.imgw.quality.qi_total")
.getData(),
"binary",
"Total quality index",
)