\documentclass[color,26pt,beamer]{IWIslides}

\usepackage{amsmath,amssymb,algorithmic}
\usepackage{multicol,verbatim}
\usepackage{amsmath,amssymb,cases}
\usepackage{graphicx,textcomp}
%\newtheorem{Def}{\color{RUGred}Definition}
%\newtheorem{Theo}{\color{RUGred}Theorem}
%\newtheorem{Prop}{\color{RUGred}Proposition}
%\newcommand{\reducopen}{\Gamma_{\subset}}
%\newcommand{\booleans}{\ensuremath{\{ 0 , 1 \}}}
%\newcommand{\card}{\ensuremath{\#}}

\definecolor{palegray}{gray}{0.7}

\hypersetup{
  pdftitle={Astro-WISE: Tracing and Using Lineage for Scientific Data Processing},
  pdfsubject={},
  pdfauthor={Johnson Mwebaze},
  pdfkeywords={},
  pdfpagemode={UseNone},
  colorlinks=true,
  linkcolor=RUGred,
  urlcolor=RUGred
  }

%\newcommand{\conclass}[1][C]{\ensuremath{\mathcal{#1}}}
%\newcommand{\powerset}[1][E]{\ensuremath{\mathcal{P}(#1)}}
%\newcommand{\imdomain}[1][M]{\ensuremath{{\mathbf #1}}}
%\newcommand{\overlap}{\ensuremath{\bot}}
%\newcommand{\id}{\ensuremath{I}}
\newcommand{\awe}{Astro-WISE}



\author{\vspace{2cm}Johnson B. Mwebaze, Danny Boxhoorn and Edwin Valentijn\\[1ex]
  \emph{Kapteyn Astronomical Institute}\\
  \emph{University of Groningen}\\
  \emph{The Netherlands}}

\title{\vspace{2cm}Astro-WISE: Tracing and Using Lineage for Scientific Data Processing}

\date{}

%\renewcommand{\movie}[1]{\href{run:#1}{\includegraphics{movie.png}}}
\definecolor{dred}{rgb}{0.894,0.0118,0.0235}
\definecolor{dgreen}{rgb}{0.0118,0.894,0.0235}
%\newcommand{\pic}[2]{\resizebox{#1\textwidth}{!}{\includegraphics{#2}}}
\newcommand{\geel}[1]{{\color{yellow}#1}}
\newcommand{\green}[1]{{\color{dgreen}#1}}
\newcommand{\red}[1]{\textbf{\color{RUGred}#1}}
\newcommand{\impl}{~\red{\ding{220}}~}


\MyLogo{NBIS 2009 / 19-21 August / Indianapolis, Indiana, USA}
\showpagenumbers{QuestionSlide}

\begin{document}
	
	\newenvironment{mylisting}
	{\begin{list}{}{\setlength{\leftmargin}{1em}}\item\small\bfseries}
	{\end{list}}

	\newenvironment{mytinylisting}
	{\begin{list}{}{\setlength{\leftmargin}{1em}}\item\tiny\bfseries}
	{\end{list}}

\newcommand{\mtlayer}[3]{
\put(0,0){\line(1,0){#1}}
\put(#1,0){\line(0,1){#2}}
\put(#1,#2){\line(-1,0){#1}}
\put(0,#2){\line(0,-1){#2}}
\put(0,0){\makebox(#1,#2){#3}}
}


\pdfcompresslevel=3

\maketitle
\raggedright

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{Outline}
  \begin{itemize}
  \item Introduction
  \item Motivation 
  \item Astro-WISE
  \item Framework
  \item Data Lineage and Data Processing
  \item Tracing Lineage in Astro-WISE
  \item Experiments [Case Studies] 
  \item Related Work
  \item Conclusions and prospects
  \end{itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Introduction}
\begin{itemize}
\item Data growing exponentially
\begin{itemize} 
\item caused by successive generations of inexpensive sensors + 
exponentially faster computing
\item Old and New Projects in the pipeline; Creating accelerated acquisition of large volumes of observational data
\end{itemize}
\item Changes the nature of scientific computing (caused by Technology, Sociology, Economics)
\begin{itemize} 
\item The Virtual Observatory,  a collection of data centers each with unique collections of astronomical data.
 \item how to believe, share and validate data as published.
\item how to selectively retrieve and process data from multiple archives interactively 
\end{itemize}
\item how to allow flexibility in e-science systems to enable researchers to work with and change methods on the fly while capturing all processing activities/events
\item Scientific computing is revolving around data [Jim Gray] - Move analysis to the data.
\end{itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Why Lineage/Provenance}
\begin{itemize}
\item Results  validation 
\item Result  debugging  
\item Reproducibility 
\item Repeatability   
\item Explanation (derivaHons, traces, proof trees) 
\item Runtime  monitoring  (Proﬁling, benchmarking) 
\item Performance  Optimization (“smart  re‐run”) 
\item Fault‐tolerance, crash‐recovery 
\item System re- design 
\end{itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{What is Astro-WISE}
\begin{itemize}
\item Goal: Connect raw data to end user in VO environment
\item an e-science infrastructure with fully distributed resources, which allows teams distributed over Europe 
\item handles optical, IR  and radio.
\begin{itemize}
\item Current: WFI@2.2m, INT, Subaru, ACS@ST  
\item Near future: Panstarrs, VST, VISTA, Lofar
\item Far future: EUCLID
\end{itemize}
\item Services
\begin{itemize}
\item pipeline data reduction, calibration and re-calibration
\item Image comparisons and combinations
\item Working with source lists
\item Visualization of astronomical data
\item publishing results to the Virtual Observatory Archives
\end{itemize}
\end{itemize}
\end{slide}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{Motivation}
\begin{itemize}
\item Wide-field imaging =  vast amounts of data

\begin{itemize}
\item VST sees equivalent of  Southern sky with 0.2” pixels  in 3 years.
\item    A project like KIDS (1000 Sq Deg) has $10^6$ 8Mpix raw data images
\end{itemize}
\item Handling of the data is non-trivial

\begin{itemize}
\item Pipeline data reduction
\item Calibration with very limited resources
\item Things change in time:
\begin{itemize}
\item Physical changes (atmosphere, various gains)
\item Code (new methods, bugs)
\item Human insight in changes
\end{itemize}   
\item Working with source lists
\end{itemize}

\end{itemize}
\end{slide}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{comment}
\begin{slide}{Virtual Observatory}
\begin{figure}[!t]
\centering
\includegraphics[scale=0.8]{images/iovo}
\end{figure}
\end{slide}
\end{comment}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{Astro-WISE}
	\begin{figure}[!t]
	\centering
	\includegraphics[scale=0.8]{images/vst}
	\end{figure}
	
\end{slide}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{AstroWise Pipeline}
\begin{figure}[!t]
\centering
\includegraphics[scale=1]{images/awpipe}
\end{figure}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{the Framework}
  \begin{itemize}
  \item The core of the system exploits three properties in a database environment. 
\begin{itemize}
\item Firstly, we apply the principle of inheritance using Object Oriented Programming (Python), where all \awe\ objects inherit key properties for database access, such as persistency of attributes. 
\item Secondly, the linking (associations or references) between instances of objects in the database is completely maintained. 
\item Thirdly, continuous growth of the database through the addition of new information or improvements made to existing information.  
  \begin{itemize}
 \item archiving  or regenerate dynamically
\item exchanging methods, scripts and configuration
  \end{itemize}
  \end{itemize}
  \end{itemize}
\end{slide}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{DataBase Design}
	\begin{itemize}
	\item Persistence - 
\begin{itemize}
\item persistence objects is the core of \awe\ pipeline processing and data lineage
\item Processing is done through invocation of methods on these objects.
\item All the I/O of the pipeline processes makes use of a database
\item Data can therefore only be manipulated through interaction with the database
\end{itemize}
\item Extendable Schema
\begin{itemize}
	\item allows extending and modifying object attributes and method definitions through inheritance and polymorphism
\end{itemize}
\item Version Control (Object Versioning) and Preservation Management
\begin{itemize}
	\item the connection between these classes and the created objects
	\item addresses object inconsistencies (mismatches) created as a result of schema modification
\end{itemize}
\end{itemize}
\end{slide}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Tracing Lineage in Astro-WISE}
\begin{itemize}
\item For each persistent class, a number of methods are defined which interact
	with the federated database. All class instantiations are automatically
	made persistent in the database forming an archive of all targets. To achieve
	this, the following major classes are implemented.
\begin{itemize}
\item \verb|DBObject| is the root class of the hierarchy of the persistent
	classes. This class defines the primary key \verb|object_id| of all objects.
\item \verb|DBObjectMeta| is the metaclass of \verb|DBObject|. This is
	the class that is responsible for class creation and object instantiation. (class factory)
\item \verb|DBProperties| is the module that defines all persistent attribute types (data dictionary)
	which are defined by \verb|persistent|.
\item \verb|DBSelect| implements a query language that is a natural extension
	to Python and that incorporates data lineage in the query syntax.
	\begin{item}
		\item $sourcelist.coadd_frame.regr_frame.reduced.raw.filename$
	\end{item}
\item \verb|DBProxy| is an abstract interface to database vendor specific
	operations.
\end{itemize}
\end{itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Lineage capture and Storage}
\begin{itemize}
\item lineage is captured as dependencies between persistent objects that refer to another persistent objects.
\item The simplest persistent class has only one persistent attribute, like
		\begin{verbatim}
		class ClassA(DBObject):
		  attribute = persistent('Description',
		           attribute_type, default_value)
		\end{verbatim}
		\item \verb|attribute| will have the type \verb|attribute_type|.
		\item Atomic attribute types, such as \verb|integer|, \verb|string| or
		\verb|float|, are translated into their database equivalents when an object
		is made persistent.
		\item If \verb|attribute_type| is a subclass of a DBObject, the attribute will be a
		link to an object of that subclass.
		\item If \verb|default_value| is an empty list, then the attribute will be an
		array of objects of \verb|attribute_type|.
		\end {itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Tracing Lineage in Astro-WISE}
	\begin{itemize}
		\item another Example
		\begin{verbatim}
		class ClassB(Object):
		  e = persistent('A link', ClassA, None)
		  f = persistent('List of links', ClassA, [])
		  g = persistent('Link to object of ClassB') 
		\end{verbatim}
		
		\item \verb|ClassB| defines three persistent attributes:
		\item \verb|e| is a link to an instance of \verb|ClassA|, 
		\item \verb|f| is a array of links to instances of \verb|ClassA| (default
		empty), and 
		\item \verb|g| is a link to another instance of \verb|ClassB|. 
		 
	\end {itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{Storage}
	\begin{itemize}
		\item To support the storage of files in a similar way, the \verb|DataObject|
		class is used.
		\begin{verbatim}
		class DataObject(DBObject):
		  filename = persistent('File part', str, '')
		\end{verbatim}
		\item The \verb|filename| attribute is used to implement \verb|store()|
		and \verb|retrieve()| methods to transfer files to and from dataservers.
		\item Because the filename is kept in the database the storage and retrieval of files is transparent to the application.
	\end {itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Target Processor}
\begin{itemize}
\item backward-chaining approach while processing data
\item The make metaphor -specify the result instead of the input
\begin{itemize}		
\item This allows the end user to trace the data product, following all its dependencies up to the raw observational data and, if necessary, to re-derive the result with better data, and/or improved methods
\end {itemize}
\item Uses target Processing (TP)
\begin{itemize}		
\item The \textit{Target} processor employs the dependency logic that is constructed using lineage data
\item Targets are rebuilt in a recursive cascade only if the dependencies have changed (i.e workflow reduction).
\end {itemize}
\end {itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{Simplified ObjectModel}
\begin{figure}[!t]
\centering
\includegraphics{images/objectmodel}
\end{figure}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{WebServcesObjectMaker}
\begin{figure}[!t]
\centering
\includegraphics[scale=0.8]{images/webServcesObjectMaker}
\end{figure}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Object Viewer - info() method}
	\begin{mytinylisting}
	\begin{verbatim}
	 +-	Name of SourceList : SL-0000010011
	 +-COMBINE_METHOD: -1
	 +-OBJECT: field07B
	 +-SLID: 10011
	 +-associatelist: -1
	 +-astrom_params: None
	 +-chip: None
	 +-creation_date: 2006-03-20 13:57:25
	 +-detection_frame: None
	 +-filename: ldac.cat
	 +-filter: <astro.main.Filter.Filter object at 0x8df22d0>
	 +-filters: 
	 +-frame: <astro.main.RegriddedFrame.CoaddedRegriddedFrame object at 0x8dd1b10>
	 +-globalname: 
	 +-instrument: <astro.main.Instrument.Instrument object at 0x8df2050>
	 +-name: SL-0000010011
	 +-number_of_sources: 2236
	 +-object_id: '0F6E9292574FC6EEE0407D81C5061B26'
	 +-process_params: <astro.main.SourceList.SourceListParameters object at 0x8df20d0>
	 +-sexconf: <astro.main.Config.SextractorConfig object at 0x8df2150>
	 +-sexparam: <class 'common.database.typed_list.typed_list'>(<type 'str'>, ['DELTA_J2000', 'ALPHA_J2000'])
	 +-sources: {}}
	 + - [some output omited]
\end{verbatim}
\end{mytinylisting}
\begin{itemize}
	\item Queries e.g sourcelist.frame.info() 
\end{itemize}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\begin{slide}{Object-Viewer}
	%\begin{itemize}
\begin{mytinylisting}
\begin{verbatim}
	Main-Object : SourceList    qualityview
	 |+ OBJECT               | STD
	 |- name                 | GAS-Sci-WVRIEND-WFI-------#842-ccd50-Red---Sci-54964.6029650-8d91ef050a6e71840937ba4a40f5381447bbc2fb.fits.slist
	 |+ number_of_sources    | 36
	 |+ sexparam             | (, ['FLUX_RADIUS', 'FWHM_IMAGE', 'MAG_ISO'])
	 | +astrom_params          [level : 2] AstrometricParameters  --> QC-WVRIEND-WFI-------#842-ccd50-AstromParams---Residual-54964.....fits
	 | | +associateconf          [level : 3] AssociateConfig    
	 | | +astromconf             [level : 3] AstromConfig    
	 | | +process_params         [level : 3] AstrometricParametersParameters    
	 | | +reduced                [level : 3] ReducedScienceFrame    qualityview
	 | | |- filename             | (image)Sci-WVRIEND-WFI-------#842-ccd50-Red---Sci-54964.6029650.....fits 
	 | | | +astrom                 [level : 4] Astrom    
	 | | | +bias                   [level : 4] BiasFrame    
	 | | | +chip                   [level : 4] Chip    
	 | | | +cold                   [level : 4] ColdPixelMap    
	 | | | +filter                 [level : 4] Filter    
	 | | | +flat                   [level : 4] MasterFlatFrame    
	 | | | +hot                    [level : 4] HotPixelMap    
	 | | | +illumination           [level : 4] IlluminationCorrectionFrame    
	 | | | +process_params         [level : 4] ReducedScienceFrameParameters    
	 | | | +raw                    [level : 4] RawScienceFrame    qualityview
	 | | | +template               [level : 4] Template    
	 | | | +weight                 [level : 4] WeightFrame    
	 | | +sexconf                [level : 3] SextractorConfig    
	 | | |- WEIGHT_IMAGE         | (image) Cal-WVRIEND-WFI-------#842-ccd50-Red---Wei-54964.6031358-....fits 
	 | |- filename             | (image)  Sci-WVRIEND-WFI-------#842-ccd50-Red---Sci-54964.6029650-....fits 		
\end{verbatim}
\end{mytinylisting}
\end{slide}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{slide}{SubImage CutOuts}
	\begin{itemize}
		\item we can trace to a pixel level
		\item this can be used to find extremely rare astronomical events, such as a fast moving star, near-earth solar system objects, or variable objects like ultra-compact binary systems or distant supernovae
	\end {itemize}
	\begin{figure}[!t]
		\centering
		\includegraphics{images/cutouts}
	\end{figure}
\end{slide}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{comment}
\begin{slide}{}
\vspace{3cm}
\begin{figure*}%[htb!]
\centering

\begin{tabular}{c c}
\begin{minipage}{12cm}
\includegraphics[scale=0.2]{images/telescope1}
\end{minipage} &

\begin{minipage}{10cm}
\includegraphics[scale=0.2]{images/telescope2}
\end{minipage} \\

\begin{minipage}{12cm}
\includegraphics[scale=0.2]{images/telescope3}
\end{minipage} &

\begin{minipage}{10cm}
\includegraphics[scale=0.2]{images/telescope4}
\end{minipage} \\

\begin{minipage}{12cm}
\includegraphics[scale=0.2]{images/lofarTelescopes}
\end{minipage} &

\begin{minipage}{10cm}
\includegraphics[scale=0.2]{images/telescope4}
\end{minipage} \\

\end{tabular}
\end{figure*}
\end{slide}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{slide}{Experiments [Case Studies] }
	\begin{center}
	    \begin{tabular}{c c }
	      \resizebox{0.5\columnwidth}{!}{\includegraphics{images/AWServices1}} &
	      \resizebox{0.5\columnwidth}{!}{\includegraphics{images/AWServices2}} \\
	      \resizebox{0.5\columnwidth}{!}{\includegraphics{images/AWServices3}} &
		\resizebox{0.5\columnwidth}{!}{\includegraphics{images/AWServices4}} 
	     
	    \end{tabular}
	  \end{center}
\end{slide}
\end{comment}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\questionslide

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%\begin{slide}{}
%  \begin{itemize}
%  \item
%  \end{itemize}
%\end{slide}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}


%%% Local Variables: 
%%% mode: latex
%%% TeX-master: t
%%% End: 
