Load files as a table of data products#

  1#!/usr/bin/env python
  2# Load files as a table of data products
  3
  4from argparse import ArgumentParser, HelpFormatter
  5import csv
  6from glob import glob
  7import os
  8import tempfile
  9import textwrap
 10
 11import firefly_client
 12
 13
 14# From https://stackoverflow.com/a/64102901
 15class RawFormatter(HelpFormatter):
 16    def _fill_text(self, text, width, indent):
 17        return "\n".join(
 18            [
 19                textwrap.fill(line, width)
 20                for line in textwrap.indent(textwrap.dedent(text), indent).splitlines()
 21            ]
 22        )
 23
 24
 25def filetable_to_firefly(
 26    ffclient, topdir, pattern, path_prefix="", recursive=True, sort=False
 27):
 28    """Upload table of files matching a pattern to a FireflyClient
 29
 30    Parameters:
 31    -----------
 32    ffclient: firefly_client.FireflyClient
 33        Instance of FireflyClient connected to a Firefly server
 34
 35    topdir: "str"
 36        pathname for directory to search
 37
 38    pattern: "str"
 39        filename pattern to search for, e.g. "*.fits"
 40
 41    path_prefix: "str"
 42        string to prepend to each file path after "file://"
 43        for example, specify "/external" for Firefly-in-Docker
 44        (default="")
 45
 46    recursive: `bool`
 47        Search all subdirectories recursively (default=True)
 48
 49    sort: `bool`
 50        Sort the file paths (default=False)
 51
 52    Returns:
 53    --------
 54    tbl_val: "str"
 55        Descriptor of table that was uploaded to the Firefly server
 56
 57    metadict: `dict`
 58        Dictionary of metadata items
 59
 60    """
 61    if recursive:
 62        file_lookup_path = os.path.join(topdir, "**", pattern)
 63    else:
 64        file_lookup_path = os.path.join(topdir, pattern)
 65    filelist = glob(file_lookup_path, recursive=recursive)
 66    if sort:
 67        filelist = sorted(filelist)
 68    metadict = {"datasource": "path"}
 69    with tempfile.NamedTemporaryFile(mode="w+t", delete=False, suffix=".csv") as fd:
 70        csv_writer = csv.writer(fd)
 71        csv_writer.writerow(["number", "name", "path"])
 72        for i, path in enumerate(filelist):
 73            # Docker Firefly allows uploads from /external
 74            csv_writer.writerow(
 75                [i, os.path.basename(path), "file://" + path_prefix + path]
 76            )
 77
 78    tbl_val = ffclient.upload_file(fd.name)
 79    os.remove(fd.name)
 80    return (tbl_val, metadict)
 81
 82
 83# Sample application
 84def main():
 85    parser = ArgumentParser(
 86        description="""
 87        Display a table of files in a Firefly window.
 88        
 89        Note that you must be running a Firefly server that is
 90        local to the data.
 91        """,
 92        epilog="""
 93        If running Firefly via Docker, note that you can mount your
 94        disk area onto /external.
 95        
 96        Sample command for running the Firefly server:
 97            docker run -p 8090:8080  -e "MAX_JVM_SIZE=64G"  \\
 98            -e "LOG_FILE_TO_CONSOLE=firefly.log" --rm --name ncmds_firefly \\
 99            -v /neoswork:/external/neoswork ipac/firefly:latest
100        """,
101        formatter_class=RawFormatter,
102    )
103    parser.add_argument("topdir", help="top-level directory to search")
104    parser.add_argument("pattern", help="filename pattern for search")
105    parser.add_argument(
106        "--path_prefix",
107        help=textwrap.dedent(
108            """string to prepend to file paths,
109        e.g. specify '/external' for Firefly-in-Docker"""
110        ),
111        default="",
112    )
113    parser.add_argument(
114        "--norecursion", help="do not recursively search topdir", action="store_true"
115    )
116    parser.add_argument("--sort", help="sort the file paths", action="store_true")
117    parser.add_argument(
118        "--firefly_url", help="URL for Firefly server", default=os.getenv("FIREFLY_URL")
119    )
120    parser.add_argument("--channel", help="channel name for websocket", default=None)
121    parser.add_argument(
122        "--html_file",
123        help="Firefly landing page (default 'firefly.html')",
124        default="firefly.html",
125    )
126    parser.add_argument(
127        "--printurl",
128        help="print browser url instead of" + " attempting to launch browser",
129        action="store_true",
130    )
131
132    args = parser.parse_args()
133    topdir = args.topdir
134    pattern = args.pattern
135    firefly_url = args.firefly_url
136    channel = args.channel
137    printurl = args.printurl
138    launch_browser = False if printurl else True
139    recursion = not args.norecursion
140    sort = args.sort
141    path_prefix = args.path_prefix
142    html_file = args.html_file
143
144    fc = firefly_client.FireflyClient.make_client(
145        url=firefly_url,
146        channel_override=channel,
147        html_file=html_file,
148        launch_browser=launch_browser,
149    )
150    if printurl:
151        print("Firefly URL is {}".format(fc.get_firefly_url()))
152
153    print("Searching for files...", end="")
154    tbl_val, metainfo = filetable_to_firefly(
155        fc, topdir, pattern, path_prefix=path_prefix, recursive=recursion, sort=sort
156    )
157    print("done.")
158
159    if printurl:
160        input("Press Enter after you have opened the Firefly URL printed above...")
161
162    # TODO: figure out how to activate data products (meta) tab in Bi-View
163    # if "slate" not in html_file:
164    #     fc.change_triview_layout(firefly_client.FireflyClient.BIVIEW_T_IChCov)
165    #     fc.dispatch('layout.updateLayout', {'images':{'selectedTab':'meta'}})
166    r = fc.add_cell(0, 0, 1, 2, "tables", "main")
167    fc.show_table(tbl_val, meta=metainfo)
168    r = fc.add_cell(0, 1, 1, 2, "tableImageMeta", "image-meta")
169    fc.show_image_metadata(viewer_id=r["cell_id"])
170
171
172if __name__ == "__main__":
173    main()

This Page