Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:modify the dataset protocol
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: 18500c968b1fc1e2a915b5c70b8cddc4a2b54de51da4eedc5454e42bfea3b479
User & Date: jack 2024-09-19 11:01:09
Context
2025-04-06
18:51
cleanup: remove noise code check-in: 5217336ae9 user: jack tags: master, trunk
2024-09-19
11:01
modify the dataset protocol check-in: 18500c968b user: jack tags: master, trunk
09:26
Update todo check-in: 82393ccdda user: jack tags: master, trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Code/dataset-and-aesthetics.lisp.

1







2
3
4
5
6








7
8
9
10





11


12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
(in-package #:eu.turtleware.polyclot.implementation)








(defgeneric dataset-head (dataset)
  (:method ((dataset list))
    (car dataset)))









(defgeneric map-dataset-points (fun dataset)
  (:method (fun (dataset list))
    (map nil fun (cdr dataset))))






(defgeneric point-values (point)


  (:method ((point sequence)) point))

;;; Creates a hash table of datasets grouped by a sequence of columns.
(defun group-by (columns dataset &key (test #'eql) (key #'identity))
  (let* ((head (dataset-head dataset))
         (result (make-hash-table :test test))
         (index* (loop for col in columns
                       collect (position col head :key key :test test))))
    (when (null index*)
      (setf (gethash nil result) dataset)
      (return-from group-by result))
    (flet ((make-ht-key-fn (row)
             (lambda (index)
               ;; when the column does not exist then the row "goes in".
               (if (null index)

>
>
>
>
>
>
>





>
>
>
>
>
>
>
>


|

>
>
>
>
>
|
>
>
|
|

|

|

|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
(in-package #:eu.turtleware.polyclot.implementation)

;;; FIXME for now we hardcode column name comparisons with EQUAL.
;;; > DATASET-COLUMN-TEST, DATASET-COLUMN-KEY ?

;;; Dataset access protocol. Contains implementation for a naive data
;;; representation based on lists, where CAR and CDR are lists of headers and
;;; rows accordingly.

(defgeneric dataset-head (dataset)
  (:method ((dataset list))
    (car dataset)))

(defgeneric dataset-body (dataset)
  (:method ((dataset list))
    (cdr dataset)))

(defgeneric map-dataset-header (fun dataset)
  (:method (fun (dataset list))
    (map nil fun (dataset-head dataset))))

(defgeneric map-dataset-points (fun dataset)
  (:method (fun (dataset list))
    (map nil fun (dataset-body dataset))))

;;; Point accessors.
(defgeneric point-index (dataset headers column-name)
  (:method ((dataset list) headers column-name)
    (position column-name headers :test #'equal)))

(defgeneric point-value (dataset point index)
  (:method ((dataset list) point index)
    (elt point index)))


;;; Creates a hash table of datasets grouped by a sequence of columns.
(defun group-by (columns dataset)
  (let* ((head (dataset-head dataset))
         (result (make-hash-table :test #'equal))
         (index* (loop for col in columns
                       collect (point-index dataset head col))))
    (when (null index*)
      (setf (gethash nil result) dataset)
      (return-from group-by result))
    (flet ((make-ht-key-fn (row)
             (lambda (index)
               ;; when the column does not exist then the row "goes in".
               (if (null index)
︙︙
56
57
58
59
60
61
62

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
                               :label "label"
                               :align-x "align-x"
                               :align-y "align-y"))

(defparameter *default-line-style* (c:make-line-style :thickness 3))
(defparameter *default-text-style* c:*default-text-style*)


(defun make-get-value-fn (dataset)
  (let ((headers (dataset-head dataset)))
    (lambda (raw-point column-name)
      (loop for val in (point-values raw-point)
            for col in headers
            when (string= col column-name)
              return val))))

;;; Try doing that in C punk! :-)
(defvar *get-value-fn*)
(defun get-value (raw-point column-name)
  (funcall *get-value-fn* raw-point column-name))

(defun make-mapping-fn (aesthetics dataset)
  (loop with head = (dataset-head dataset)
        with cont = (gensym "CONT")
        with raw-point = (gensym "RAW-POINT")
        with point-values = nil
        with position = nil
        for (aesthetic source) on aesthetics by #'cddr
        when (stringp source) do
          ;; Ignore aesthetics that could not be mapped.
          (setf position (position source head :test #'equal))
          (if position
              (unless point-values
                (setf point-values (gensym "POINT-VALUES")))
              (setf source nil))
        unless (null source)
          appending (list aesthetic (typecase source
                                      (string `(elt ,point-values ,position))
                                      (function `(funcall ,source ,raw-point))
                                      (otherwise source)))
            into lambda-list
        finally
           (return
             (let ((fun (if (null point-values)
                            `(lambda (,cont ,raw-point)
                               (declare (ignore ,raw-point))
                               (funcall ,cont ,@lambda-list))
                            `(lambda (,cont ,raw-point)
                               (let ((,point-values (point-values ,raw-point)))
                                 (funcall ,cont ,@lambda-list))))))
               (values (compile nil fun) fun)))))

;;; MAYBE currently we access dataset-head and then map points and use
;;; point-values. Perhaps a better approach will be passing header and element
;;; accessor as functions.
(defun bind-data (aesthetics dataset &key (class <mapped-data>))
  (make-instance class :dataset dataset :aesthetics aesthetics))

;;; Binding data conses a new vector so it may be potentially space and time
;;; consuming operation. We hide that behind the protocol so an alternate
;;; appraoch is possible.
(define-class <mapped-data> ()







>


|
<
<
|
|



|
|





<
|



|
<
|
<
|


|





<
|
<
|
<
<
<


<
<
<







78
79
80
81
82
83
84
85
86
87
88


89
90
91
92
93
94
95
96
97
98
99
100

101
102
103
104
105

106

107
108
109
110
111
112
113
114
115

116

117



118
119



120
121
122
123
124
125
126
                               :label "label"
                               :align-x "align-x"
                               :align-y "align-y"))

(defparameter *default-line-style* (c:make-line-style :thickness 3))
(defparameter *default-text-style* c:*default-text-style*)

;;; 
(defun make-get-value-fn (dataset)
  (let ((headers (dataset-head dataset)))
   (lambda (point column-name)


     (let ((index (point-index dataset headers column-name)))
       (point-value dataset point index)))))

;;; Try doing that in C punk! :-)
(defvar *get-value-fn*)
(defun get-value (point column-name)
  (funcall *get-value-fn* point column-name))

(defun make-mapping-fn (aesthetics dataset)
  (loop with head = (dataset-head dataset)
        with cont = (gensym "CONT")
        with raw-point = (gensym "RAW-POINT")

        with index = nil
        for (aesthetic source) on aesthetics by #'cddr
        when (stringp source) do
          ;; Ignore aesthetics that could not be mapped.
          (setf index (point-index dataset head source))

          (unless index

            (setf source nil))
        unless (null source)
          appending (list aesthetic (typecase source
                                      (string `(point-value ',dataset ,raw-point ,index))
                                      (function `(funcall ,source ,raw-point))
                                      (otherwise source)))
            into lambda-list
        finally
           (return

             (let ((fun `(lambda (,cont ,raw-point)

                           (funcall ,cont ,@lambda-list))))



               (values (compile nil fun) fun)))))




(defun bind-data (aesthetics dataset &key (class <mapped-data>))
  (make-instance class :dataset dataset :aesthetics aesthetics))

;;; Binding data conses a new vector so it may be potentially space and time
;;; consuming operation. We hide that behind the protocol so an alternate
;;; appraoch is possible.
(define-class <mapped-data> ()
︙︙
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
                     :ink* c:+transparent-ink+
                     :line-style *default-line-style*
                     :text-style *default-text-style*
                     :label nil
                     :align-x :center
                     :align-y :center))

(defmethod dataset-head ((object <mapped-data>))
  (dataset-head (dataset object)))

(defun show-dataset (dataset stream)
  (flet ((do-it (*standard-output*)
           (c:formatting-table (t :x-spacing 30)
             (c:with-drawing-options (t :text-face :bold)
               (c:surrounding-output-with-border (t :padding 0 :ink c:+light-pink+ :filled t)
                 (c:formatting-row ()
                   (c:formatting-cell () (format t "()"))







<
<
<







139
140
141
142
143
144
145



146
147
148
149
150
151
152
                     :ink* c:+transparent-ink+
                     :line-style *default-line-style*
                     :text-style *default-text-style*
                     :label nil
                     :align-x :center
                     :align-y :center))




(defun show-dataset (dataset stream)
  (flet ((do-it (*standard-output*)
           (c:formatting-table (t :x-spacing 30)
             (c:with-drawing-options (t :text-face :bold)
               (c:surrounding-output-with-border (t :padding 0 :ink c:+light-pink+ :filled t)
                 (c:formatting-row ()
                   (c:formatting-cell () (format t "()"))
︙︙
168
169
170
171
172
173
174



175
176
177
178
179
180
181
182
183
184
185
186
(defun dataset-column (mapped-dataset column-name)
  (let* ((nrows (length (data-array mapped-dataset)))
         (vector (make-array (list nrows) :fill-pointer 0)))
    (do-points (point mapped-dataset)
      (vector-push-extend (slot-value point column-name) vector))
    vector))




(defmethod map-dataset-points (fun (object <mapped-data>))
  (map nil fun (data-array object)))

(defmethod initialize-instance :after ((obj <mapped-data>) &key dataset aesthetics)
  (let ((*get-value-fn* (make-get-value-fn dataset))
        (mapping-fn (make-mapping-fn aesthetics dataset)))
    (let ((array (make-array '(0) :adjustable t :fill-pointer t)))
      (do-points (point dataset)
        (vector-push-extend (funcall mapping-fn #'<row> point) array))
      (setf (get-value-fn obj) *get-value-fn*
            (mapping-fn obj) mapping-fn
            (data-array obj) array))))







>
>
>












175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
(defun dataset-column (mapped-dataset column-name)
  (let* ((nrows (length (data-array mapped-dataset)))
         (vector (make-array (list nrows) :fill-pointer 0)))
    (do-points (point mapped-dataset)
      (vector-push-extend (slot-value point column-name) vector))
    vector))

;;; FIXME implement the rest of the dataset protocol for <MAPPED-DATA>; column
;;; names and indexes should be based on *DEFAULT-AESTHETICS* variable (in this
;;; case the index would be a slot name). It is not a big deal now.
(defmethod map-dataset-points (fun (object <mapped-data>))
  (map nil fun (data-array object)))

(defmethod initialize-instance :after ((obj <mapped-data>) &key dataset aesthetics)
  (let ((*get-value-fn* (make-get-value-fn dataset))
        (mapping-fn (make-mapping-fn aesthetics dataset)))
    (let ((array (make-array '(0) :adjustable t :fill-pointer t)))
      (do-points (point dataset)
        (vector-push-extend (funcall mapping-fn #'<row> point) array))
      (setf (get-value-fn obj) *get-value-fn*
            (mapping-fn obj) mapping-fn
            (data-array obj) array))))

Changes to Code/faceting.lisp.

︙︙
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
  (group-by nil dataset))

;;; "Ribbon" (one variable)
(define-class <faceting-wrap> (<faceting>)
  (var nrow ncol direction))

(defmethod compute-datasets ((faceting <faceting-wrap>) dataset)
  (group-by (var faceting) dataset :test #'equal))

;;; "Grid" (two variables)
(define-class <faceting-grid> (<faceting>)
  (var1 var2))

(defmethod compute-datasets ((faceting <faceting-grid>) dataset)
  (group-by (list (var1 faceting)
                  (var2 faceting))
            dataset
            :test #'equal))



;;; There may be more than one <facet> per panel - each distinct dataset implies
;;; a separate object for the set of variables. That means that we _can_ have
;;; many incongruent scales on the same render (and that's a good thing!).








|








|
<







28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

45
46
47
48
49
50
51
  (group-by nil dataset))

;;; "Ribbon" (one variable)
(define-class <faceting-wrap> (<faceting>)
  (var nrow ncol direction))

(defmethod compute-datasets ((faceting <faceting-wrap>) dataset)
  (group-by (var faceting) dataset))

;;; "Grid" (two variables)
(define-class <faceting-grid> (<faceting>)
  (var1 var2))

(defmethod compute-datasets ((faceting <faceting-grid>) dataset)
  (group-by (list (var1 faceting)
                  (var2 faceting))
            dataset))




;;; There may be more than one <facet> per panel - each distinct dataset implies
;;; a separate object for the set of variables. That means that we _can_ have
;;; many incongruent scales on the same render (and that's a good thing!).

︙︙

Changes to README.org.

︙︙
173
174
175
176
177
178
179
180
181
182
183
184
185


186
187
188
189
190
191
192
should be for now considered to be destructive. Finally we estabilish local
coordinates with the macro ~with-coords~ and call ~plot-geom~ which goes through
the dataset and does the drawing.

ToDo:

- [X] <cartesian> defaults to the screen orientation (should be graphics)
- [ ] dataset and aesthetics
  - [ ] <row> is an instance with fixed slots, we want to extend it (a vector?)
  - [ ] WITH-AES assumes an instance with particualr slots
    We'd like to make it specializable
  - [ ] POINT-VALUES returns a sequence, so for non-sequence rows we are consing
    Think of an alternative protocol that extracts only referenced columns.


- [-] all geoms are very primitive often not even using relevant aes
  - [X] <geom-point>
  - [X] <geom-line>
  - [X] <geom-bar>
  - [ ] provide extra polishing(!)
- [7/9] implementation of scales and guides
  - [X] basic framework for scales is in place already







|



|

>
>







173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
should be for now considered to be destructive. Finally we estabilish local
coordinates with the macro ~with-coords~ and call ~plot-geom~ which goes through
the dataset and does the drawing.

ToDo:

- [X] <cartesian> defaults to the screen orientation (should be graphics)
- [-] dataset and aesthetics
  - [ ] <row> is an instance with fixed slots, we want to extend it (a vector?)
  - [ ] WITH-AES assumes an instance with particualr slots
    We'd like to make it specializable
  - [X] POINT-VALUES returns a sequence, so for non-sequence rows we are consing
    Think of an alternative protocol that extracts only referenced columns.
    POINT-VALUES is removed and POINT-INDEX and POINT-VALUE are introduced
    instedad.
- [-] all geoms are very primitive often not even using relevant aes
  - [X] <geom-point>
  - [X] <geom-line>
  - [X] <geom-bar>
  - [ ] provide extra polishing(!)
- [7/9] implementation of scales and guides
  - [X] basic framework for scales is in place already
︙︙
252
253
254
255
256
257
258

259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276












277
278
279
280
281
282
283
- [2022-11-16 Wed] implemented binned scale (and fixed scale mapping)
- [2022-12-11 nie] improvements to scales, a simple listener tool (demo)
- [2022-12-13 wto] add colors, initial render of the color scale
- [2022-12-14 śro] position adjustment (dodge)
- [2023-05-17 Wed] the architecture has been changed to account for faceting
- [2024-05-08 Wed] resuming development (almost a year! In train after els'24)
- [2024-09-19 Thu] when mapping allow access by user function to column by name


** Dataset and aesthetics

Currently we allow for the user to map aesthetics by name (a string), arbitrary
function (must be a function, not a function designator), and as a constant.

When the user passes a function, then it should accept as an argument raw
point. This function can call in its dynamic context the function GET-VALUE.
For example

#+begin_src lisp
  :aesthetics (list :x "x"
                    :y "y"
                    :ink "ink"
                    :line-style (lambda (raw-point)
                                  (let ((x (get-value raw-point "x")))
                                    (make-line-style :thickness (/ (sqrt (abs x)) 3)))))
#+end_src













** Faceting

Facets are being implemented after a single plot abstraction has been defined
and that demands some architectural changes. Let's start with a description:

Faceting is an operation of splitting datasets into a few separate sets that are







>


















>
>
>
>
>
>
>
>
>
>
>
>







254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
- [2022-11-16 Wed] implemented binned scale (and fixed scale mapping)
- [2022-12-11 nie] improvements to scales, a simple listener tool (demo)
- [2022-12-13 wto] add colors, initial render of the color scale
- [2022-12-14 śro] position adjustment (dodge)
- [2023-05-17 Wed] the architecture has been changed to account for faceting
- [2024-05-08 Wed] resuming development (almost a year! In train after els'24)
- [2024-09-19 Thu] when mapping allow access by user function to column by name
- [2024-09-19 Thu] replace POINT-VALUES with POINT-INDEX and POINT-VALUE

** Dataset and aesthetics

Currently we allow for the user to map aesthetics by name (a string), arbitrary
function (must be a function, not a function designator), and as a constant.

When the user passes a function, then it should accept as an argument raw
point. This function can call in its dynamic context the function GET-VALUE.
For example

#+begin_src lisp
  :aesthetics (list :x "x"
                    :y "y"
                    :ink "ink"
                    :line-style (lambda (raw-point)
                                  (let ((x (get-value raw-point "x")))
                                    (make-line-style :thickness (/ (sqrt (abs x)) 3)))))
#+end_src

To allow Polyclot access the dataset, the producer is expected to specialize
functions:

(dataset-head dataset)                                ; #<opaque-header>
(dataset-body dataset)                                ; #<opaque-points>

(map-dataset-header fun dataset) ; map fun over dataset header (columns)
(map-dataset-points fun dataset) ; map fun over dataset points (data rows)

(point-index dataset #<opaque-header> column-name)     ; #<opaque-index>
(point-value dataset #<opaque-points> #<opaque-index>) ; aesthetic-value

** Faceting

Facets are being implemented after a single plot abstraction has been defined
and that demands some architectural changes. Let's start with a description:

Faceting is an operation of splitting datasets into a few separate sets that are
︙︙