go_utils.lc

  1import math
  2import re
  3
  4import matplotlib.pyplot as plt
  5import numpy as np
  6import pandas as pd
  7import seaborn as sns
  8
  9from go_utils.cleanup import (
 10    camel_case,
 11    remove_homogenous_cols,
 12    rename_latlon_cols,
 13    replace_column_prefix,
 14    round_cols,
 15    standardize_null_vals,
 16)
 17from go_utils.plot import completeness_histogram, multiple_bar_graph, plot_freq_bar
 18
 19__doc__ = """
 20
 21## Unpacking the Landcover Classification Data
 22The classification data for each entry is condensed into several entries separated by a semicolon. [This method](#unpack_classifications) identifies and parses Land Cover Classifications and percentages to create new columns. The columns are also reordered to better group directional information together.
 23
 24The end result is a DataFrame that contains columns for every Unique Landcover Classification (per direction) and its respective percentages for each entry.
 25
 26There are four main steps to this procedure:
 271.Identifying Land Cover Classifications for each Cardinal Direction: An internal method returns the unique description (e.g. HerbaceousGrasslandTallGrass) listed in a column. This method is run for all 4 cardinal directions to obtain the all unique classifications per direction.
 282. Creating empty columns for each Classification from each Cardinal Direction: Using the newly identified classifications new columns are made for each unique classification. These columns initially contained the default float64 value of 0.0. By initializing all the classification column values to 0.0, we ensure no empty values are set to -9999 in the round_cols(df) method (discussed in General Cleanup Procedures - Round Appropriate Columns). This step eases future numerical analysis.
 293. Grouping and Alphabetically Sorting Directional Column Information: To better organize the DataFrame, columns containing any of the following directional substrings: "downward", "upward", "west", "east", "north", "south" (case insensitive) are identified and alphabetically sorted. Then an internal method called move_cols, specified column headers to move (direction_data_cols), and the location before the desired point of insertion, the program returns a reordered DataFrame, where all directional columns are grouped together. This greatly improves the Land Covers dataset’s organization and accessibility.
 304. Adding Classification Percentages to their respective Land Cover Classification Columns - To fill in each classification column with their respective percentages, an internal method is applied to each row in the dataframe. This method iterates through each classification direction (ie “lc_EastClassifications”) and sets each identified Classification column with its respective percentage.
 31
 32NOTE: After these procedures, the original directional classification columns (e.g. “lc_EastClassifications”) are not dropped.
 33"""
 34
 35classifications = []
 36
 37
 38def cleanup_column_prefix(df, inplace=False):
 39    """Method for shortening raw landcover column names.
 40
 41    The df object will now replace the verbose `landcovers` prefix in some of the columns with `lc_`
 42
 43    Parameters
 44    ----------
 45    df : pd.DataFrame
 46        The DataFrame containing raw landcover data. The DataFrame object itself will be modified.
 47    inplace : bool, default=False
 48        Whether to return a new DataFrame. If True then no DataFrame copy is not returned and the operation is performed in place.
 49
 50    Returns
 51    -------
 52    pd.DataFrame or None
 53        A DataFrame with the cleaned up column prefixes. If `inplace=True` it returns None.
 54    """
 55
 56    if not inplace:
 57        df = df.copy()
 58
 59    replace_column_prefix(df, "landcovers", "lc", inplace=True)
 60
 61    if not inplace:
 62        return df
 63
 64
 65def extract_classification_name(entry):
 66    """
 67    Extracts the name (landcover description) of a singular landcover classification. For example in the classification of `"60% MUC 02 (b) [Trees, Closely Spaced, Deciduous - Broad Leaved]"`, the `"Trees, Closely Spaced, Deciduous - Broad Leaved"` is extracted.
 68
 69    Parameters
 70    ----------
 71    entry : str
 72        A single landcover classification.
 73
 74    Returns
 75    -------
 76    str
 77        The Landcover description of a classification
 78    """
 79
 80    return re.search(r"(?<=\[).*(?=\])", entry).group()
 81
 82
 83def extract_classification_percentage(entry):
 84    """
 85    Extracts the percentage of a singular landcover classification. For example in the classification of `"60% MUC 02 (b) [Trees, Closely Spaced, Deciduous - Broad Leaved]"`, the `60` is extracted.
 86
 87    Parameters
 88    ----------
 89    entry : str
 90        A single landcover classification.
 91
 92    Returns
 93    -------
 94    float
 95        The percentage of a landcover classification
 96    """
 97
 98    return float(re.search(".*(?=%)", entry).group())
 99
100
101def _extract_landcover_items(func, info):
102    entries = info.split(";")
103    return [func(entry) for entry in entries]
104
105
106def extract_classifications(info):
107    """Extracts the name/landcover description (see [here](#extract_classification_name) for a clearer definition) of a landcover classification entry in the GLOBE Observer Data.
108
109    Parameters
110    ----------
111    info : str
112        A string representing a landcover classification entry in the GLOBE Observer Datset.
113
114    Returns
115    -------
116    list of str
117        The different landcover classifications stored within the landcover entry.
118    """
119    return _extract_landcover_items(extract_classification_name, info)
120
121
122def extract_percentages(info):
123    """Extracts the percentages (see [here](#extract_classification_percentage) for a clearer definition) of a landcover classification in the GLOBE Observer Datset.
124
125    Parameters
126    ----------
127    info : str
128        A string representing a landcover classification entry in the GLOBE Observer Datset.
129
130    Returns
131    -------
132    list of float
133        The different landcover percentages stored within the landcover entry.
134    """
135
136    return _extract_landcover_items(extract_classification_percentage, info)
137
138
139def extract_classification_dict(info):
140    """Extracts the landcover descriptions and percentages of a landcover classification entry as a dictionary.
141
142    Parameters
143    ----------
144    info : str
145        A string representing a landcover classification entry in the GLOBE Observer Datset.
146
147    Returns
148    -------
149    dict of str, float
150        The landcover descriptions and percentages stored as a dict in the form: `{"description" : percentage}`.
151    """
152
153    entries = info.split(";")
154    return {
155        extract_classification_name(entry): extract_classification_percentage(entry)
156        for entry in entries
157    }
158
159
160def _get_classifications_for_direction(df, direction_col_name):
161    list_of_land_types = []
162    for info in df[direction_col_name]:
163        # Note: Sometimes info = np.nan, a float -- In that case we do NOT parse/split
164        if type(info) == str:
165            [
166                list_of_land_types.append(camel_case(entry, [" ", ",", "-", "/"]))
167                for entry in extract_classifications(info)
168            ]
169    return np.unique(list_of_land_types).tolist()
170
171
172def _move_cols(df, cols_to_move=[], ref_col=""):
173    col_names = df.columns.tolist()
174    index_before_desired_loc = col_names.index(ref_col)
175
176    cols_before_index = col_names[: index_before_desired_loc + 1]
177    cols_at_index = cols_to_move
178
179    cols_before_index = [i for i in cols_before_index if i not in cols_at_index]
180    cols_after_index = [
181        i for i in col_names if i not in cols_before_index + cols_at_index
182    ]
183
184    return df[cols_before_index + cols_at_index + cols_after_index]
185
186
187def unpack_classifications(
188    lc_df,
189    north="lc_NorthClassifications",
190    east="lc_EastClassifications",
191    south="lc_SouthClassifications",
192    west="lc_WestClassifications",
193    ref_col="lc_pid",
194    unpack=True,
195):
196    """
197    Unpacks the classification data in the *raw* GLOBE Observer Landcover data. This method assumes that the columns have been renamed with accordance to the [column cleanup](#cleanup_column_prefix) method.
198
199    This returns a copy of the dataframe.
200
201    See [here](#unpacking-the-landcover-classification-data) for more information.
202
203    *Note:* The returned DataFrame will have around 250 columns.
204
205    Parameters
206    ----------
207    lc_df : pd.DataFrame
208        A DataFrame containing Raw GLOBE Observer Landcover data that has had the column names simplified.
209    north: str, default="lc_NorthClassifications"
210        The name of the column which contains the North Classifications
211    east: str, default="lc_EastClassifications"
212        The name of the column which contains the East Classifications
213    south: str, default="lc_SouthClassifications"
214        The name of the column which contains the South Classifications
215    west: str, default="lc_WestClassifications"
216        The name of the column which contains the West Classifications
217    ref_col: str, default="lc_pid"
218        The name of the column which all of the expanded values will be placed after. For example, if the columns were `[1, 2, 3, 4]` and you chose 3, the new columns will now be `[1, 2, 3, (all classification columns), 4]`.
219    unpack: bool, default=True
220        True if you want to unpack the directional classifications, False if you only want overall classifications
221
222    Returns
223    -------
224    pd.DataFrame
225        A DataFrame with the unpacked classification columns.
226    list
227        A list containing all the generated overall Land Cover column names (mainly for testing purposes).
228    list
229        A list containing all the generated directional Land Cover column names (mainly for testing purposes).
230    """
231
232    classifications = [north, east, south, west]
233
234    def set_directions(row):
235        for classification in classifications:
236            if not pd.isnull(row[classification]):
237                entries = row[classification].split(";")
238                for entry in entries:
239                    percent, name = (
240                        extract_classification_percentage(entry),
241                        extract_classification_name(entry),
242                    )
243                    name = camel_case(name, [" ", ",", "-", "/"])
244                    classification = classification.replace("Classifications", "_")
245                    overall = re.sub(
246                        r"(north|south|east|west).*",
247                        "Overall_",
248                        key,
249                        flags=re.IGNORECASE,
250                    )
251                    row[f"{classification}{name.strip()}"] = percent
252                    row[f"{overall}{name.strip()}"] += percent
253        return row
254
255    land_type_columns_to_add = {
256        classification: _get_classifications_for_direction(lc_df, classification)
257        for classification in classifications
258    }
259    overall_columns = set()
260    direction_cols = set()
261    for key, values in land_type_columns_to_add.items():
262        direction_name = key.replace("Classifications", "_")
263        overall = re.sub(
264            r"(north|south|east|west).*", "Overall_", key, flags=re.IGNORECASE
265        )
266        for value in values:
267            direction_cols.add(direction_name + value)
268            overall_columns.add(overall + value)
269    overall_columns = list(overall_columns)
270    direction_cols = list(direction_cols)
271    direction_data_cols = sorted(overall_columns + direction_cols)
272
273    # Creates a blank DataFrame and concats it to the original to avoid iteratively growing the LC DataFrame
274    blank_df = pd.DataFrame(
275        np.zeros((len(lc_df), len(direction_data_cols))), columns=direction_data_cols
276    )
277
278    lc_df = pd.concat([lc_df, blank_df], axis=1)
279
280    lc_df = _move_cols(lc_df, cols_to_move=direction_data_cols, ref_col=ref_col)
281    lc_df = lc_df.apply(set_directions, axis=1)
282    for column in overall_columns:
283        lc_df[column] /= 4
284
285    if not unpack:
286        lc_df = lc_df.drop(columns=direction_cols)
287    return lc_df, overall_columns, direction_cols
288
289
290def photo_bit_flags(
291    df,
292    up="lc_UpwardPhotoUrl",
293    down="lc_DownwardPhotoUrl",
294    north="lc_NorthPhotoUrl",
295    south="lc_SouthPhotoUrl",
296    east="lc_EastPhotoUrl",
297    west="lc_WestPhotoUrl",
298    photo_count="lc_PhotoCount",
299    rejected_count="lc_RejectedCount",
300    pending_count="lc_PendingCount",
301    empty_count="lc_EmptyCount",
302    bit_binary="lc_PhotoBitBinary",
303    bit_decimal="lc_PhotoBitDecimal",
304    inplace=False,
305):
306    """
307    Creates the following flags:
308    - `PhotoCount`: The number of valid photos per record.
309    - `RejectedCount`: The number of photos that were rejected per record.
310    - `PendingCount`: The number of photos that are pending approval per record.
311    - `PhotoBitBinary`: A string that represents the presence of a photo in the Up, Down, North, South, East, and West directions. For example, if the entry is `110100`, that indicates that there is a valid photo for the Up, Down, and South Directions but no valid photos for the North, East, and West Directions.
312    - `PhotoBitDecimal`: The numerical representation of the lc_PhotoBitBinary string.
313
314    Parameters
315    ----------
316    df : pd.DataFrame
317        A land cover DataFrame
318    up : str, default="lc_UpwardPhotoUrl"
319        The name of the column in the land cover DataFrame that contains the url for the upwards photo.
320    down : str, default="lc_DownwardPhotoUrl"
321        The name of the column in the land cover DataFrame that contains the url for the downwards photo.
322    north : str, default="lc_NorthPhotoUrl"
323        The name of the column in the land cover DataFrame that contains the url for the north photo.
324    south : str, default="lc_SouthPhotoUrl"
325        The name of the column in the land cover DataFrame that contains the url for the south photo.
326    east : str, default="lc_EastPhotoUrl"
327        The name of the column in the land cover DataFrame that contains the url for the east photo.
328    west : str, default="lc_WestPhotoUrl"
329        The name of the column in the land cover DataFrame that contains the url for the west photo.
330    photo_count : str, default="lc_PhotoCount"
331        The name of the column that will be storing the PhotoCount flag.
332    rejected_count : str, default="lc_RejectedCount"
333        The name of the column that will be storing the RejectedCount flag.
334    pending_count : str, default="lc_PendingCount"
335        The name of the column that will be storing the PendingCount flag.
336    empty_count : str, default="lc_EmptyCount"
337        The name of the column that will be storing the EmptyCount flag.
338    bit_binary : str, default="lc_PhotoBitBinary"
339        The name of the column that will be storing the PhotoBitBinary flag.
340    bit_decimal : str, default="lc_PhotoBitDecimal"
341        The name of the column that will be storing the PhotoBitDecimal flag.
342    inplace : bool, default=False
343        Whether to return a new DataFrame. If True then no DataFrame copy is not returned and the operation is performed in place.
344
345    Returns
346    -------
347    pd.DataFrame or None
348        A DataFrame with the photo bit flags. If `inplace=True` it returns None.
349    """
350
351    def pic_data(*args):
352        pic_count = 0
353        rejected_count = 0
354        pending_count = 0
355        empty_count = 0
356        valid_photo_bit_mask = ""
357
358        for entry in args:
359            if not pd.isna(entry) and "http" in entry:
360                valid_photo_bit_mask += "1"
361                pic_count += entry.count("http")
362            else:
363                valid_photo_bit_mask += "0"
364            if pd.isna(entry):
365                empty_count += 1
366            else:
367                pending_count += entry.count("pending")
368                rejected_count += entry.count("rejected")
369        return (
370            pic_count,
371            rejected_count,
372            pending_count,
373            empty_count,
374            valid_photo_bit_mask,
375            int(valid_photo_bit_mask, 2),
376        )
377
378    if not inplace:
379        df = df.copy()
380
381    get_photo_data = np.vectorize(pic_data)
382    (
383        df[photo_count],
384        df[rejected_count],
385        df[pending_count],
386        df[empty_count],
387        df[bit_binary],
388        df[bit_decimal],
389    ) = get_photo_data(
390        df[up].to_numpy(),
391        df[down].to_numpy(),
392        df[north].to_numpy(),
393        df[south].to_numpy(),
394        df[east].to_numpy(),
395        df[west].to_numpy(),
396    )
397
398    if not inplace:
399        return df
400
401
402def classification_bit_flags(
403    df,
404    north="lc_NorthClassifications",
405    south="lc_SouthClassifications",
406    east="lc_EastClassifications",
407    west="lc_WestClassifications",
408    classification_count="lc_ClassificationCount",
409    bit_binary="lc_ClassificationBitBinary",
410    bit_decimal="lc_ClassificationBitDecimal",
411    inplace=False,
412):
413    """
414    Creates the following flags:
415    - `ClassificationCount`: The number of classifications per record.
416    - `BitBinary`: A string that represents the presence of a classification in the North, South, East, and West directions. For example, if the entry is `1101`, that indicates that there is a valid classification for the North, South, and West Directions but no valid classifications for the East Direction.
417    - `BitDecimal`: The number of photos that are pending approval per record.
418
419    Parameters
420    ----------
421    df : pd.DataFrame
422        A land cover DataFrame
423    north : str, default="lc_NorthClassifications"
424        The name of the column in the land cover DataFrame that contains the north classification.
425    south : str, default="lc_SouthClassifications"
426        The name of the column in the land cover DataFrame that contains the south classification.
427    east : str, default="lc_EastClassifications"
428        The name of the column in the land cover DataFrame that contains the east classification.
429    west : str, default="lc_WestClassifications"
430        The name of the column in the land cover DataFrame that contains the west classification.
431    classification_count : str, default="lc_ClassificationCount"
432        The name of the column that will store the ClassificationCount flag.
433    bit_binary : str, default="lc_ClassificationBitBinary"
434        The name of the column that will store the BitBinary flag.
435    bit_decimal : str, default="lc_ClassificationBitDecimal"
436        The name of the column that will store the BitDecimal flag.
437    inplace : bool, default=False
438        Whether to return a new DataFrame. If True then no DataFrame copy is not returned and the operation is performed in place.
439
440    Returns
441    -------
442    pd.DataFrame or None
443        A DataFrame with the classification bit flags. If `inplace=True` it returns None.
444    """
445
446    def classification_data(*args):
447        classification_count = 0
448        classification_bit_mask = ""
449        for entry in args:
450            if pd.isna(entry) or entry is np.nan:
451                classification_bit_mask += "0"
452            else:
453                classification_count += 1
454                classification_bit_mask += "1"
455        return (
456            classification_count,
457            classification_bit_mask,
458            int(classification_bit_mask, 2),
459        )
460
461    if not inplace:
462        df = df.copy()
463    get_classification_data = np.vectorize(classification_data)
464
465    (
466        df[classification_count],
467        df[bit_binary],
468        df[bit_decimal],
469    ) = get_classification_data(
470        df[north],
471        df[south],
472        df[east],
473        df[west],
474    )
475    if not inplace:
476        return df
477
478
479def completion_scores(
480    df,
481    photo_bit_binary="lc_PhotoBitBinary",
482    classification_binary="lc_ClassificationBitBinary",
483    sub_completeness="lc_SubCompletenessScore",
484    completeness="lc_CumulativeCompletenessScore",
485    inplace=False,
486):
487    """
488    Adds the following completness score flags:
489    - `SubCompletenessScore`: The percentage of valid landcover classifications and photos that are filled out.
490    - `CumulativeCompletenessScore`: The percentage of non null values out of all the columns.
491
492    Parameters
493    ----------
494    df : pd.DataFrame
495        A landcover DataFrame with the [`PhotoBitBinary`](#photo_bit_flags) and [`ClassificationBitBinary`](#classification_bit_flags) flags.
496    photo_bit_binary : str, default="lc_PhotoBitBinary"
497        The name of the column that stores the PhotoBitBinary flag.
498    classification_binary : str, default="lc_PhotoBitBinary"
499        The name of the column that stores the ClassificationBitBinary flag.
500    sub_completeness : str, default="lc_PhotoBitBinary"
501        The name of the column that will store the generated SubCompletenessScore flag.
502    completeness : str, default="lc_PhotoBitBinary"
503        The name of the column that will store the generated CompletenessScore flag.
504    inplace : bool, default=False
505        Whether to return a new DataFrame. If True then no DataFrame copy is not returned and the operation is performed in place.
506
507    Returns
508    -------
509    pd.DataFrame or None
510        A DataFrame with the completeness score flags. If `inplace=True` it returns None.
511    """
512
513    def sum_bit_mask(bit_mask="0"):
514        sum = 0.0
515        for char in bit_mask:
516            sum += int(char)
517        return sum
518
519    if not inplace:
520        df = df.copy()
521
522    scores = {}
523    scores["sub_score"] = []
524    # Cummulative Completion Score
525    scores["cumulative_score"] = round(df.count(1) / len(df.columns), 2)
526    # Sub-Score
527    for index in df.index:
528        bit_mask = df[photo_bit_binary][index] + df[classification_binary][index]
529        sub_score = round(sum_bit_mask(bit_mask=bit_mask), 2)
530        sub_score /= len(bit_mask)
531        scores["sub_score"].append(sub_score)
532
533    df[sub_completeness], df[completeness] = (
534        scores["sub_score"],
535        scores["cumulative_score"],
536    )
537
538    if not inplace:
539        return df
540
541
542def apply_cleanup(lc_df, unpack=True):
543    """Applies a full cleanup procedure to the landcover data.
544    It follows the following steps:
545    - Removes Homogenous Columns
546    - Renames Latitude and Longitudes
547    - Cleans the Column Naming
548    - Unpacks landcover classifications
549    - Rounds Columns
550    - Standardizes Null Values
551
552    This returns a copy
553
554    Parameters
555    ----------
556    lc_df : pd.DataFrame
557        A DataFrame containing **raw** Landcover Data from the API.
558    unpack : bool
559        If True, the Landcover data will expand the classifications into separate columns (results in around 300 columns). If False, it will just unpack overall landcover.
560
561    Returns
562    -------
563    pd.DataFrame
564        A DataFrame containing the cleaned Landcover Data
565    """
566    lc_df = lc_df.copy()
567
568    remove_homogenous_cols(lc_df, inplace=True)
569    rename_latlon_cols(lc_df, inplace=True)
570    cleanup_column_prefix(lc_df, inplace=True)
571    lc_df, overall_cols, directional_cols = unpack_classifications(lc_df, unpack=unpack)
572
573    round_cols(lc_df, inplace=True)
574    standardize_null_vals(lc_df, inplace=True)
575    return lc_df
576
577
578def add_flags(lc_df):
579    """Adds the following flags to the landcover data:
580    - Photo Bit Flags
581    - Classification Bit Flags
582    - Completeness Score Flags
583
584    Returns a copy of the DataFrame
585
586    Parameters
587    ----------
588    lc_df : pd.DataFrame
589        A DataFrame containing cleaned up Landcover Data ideally from the [apply_cleanup](#apply_cleanup) method.
590
591    Returns
592    -------
593    pd.DataFrame
594        A DataFrame containing the Land Cover flags.
595    """
596    lc_df = lc_df.copy()
597    photo_bit_flags(lc_df, inplace=True)
598    classification_bit_flags(lc_df, inplace=True)
599    get_main_classifications(lc_df, inplace=True)
600    completion_scores(lc_df, inplace=True)
601    return lc_df
602
603
604def direction_frequency(lc_df, direction_list, bit_binary, entry_type):
605    """
606    Plots the amount of a variable of interest for each direction.
607
608    Parameters
609    ----------
610    lc_df : pd.DataFrame
611        The DataFrame containing Land Cover Data.
612    direction_list : list of str
613        The column names of the different variables of interest for each direction.
614    bit_binary: str
615        The Bit Binary Flag associated with the variable of interest.
616    entry_type: str
617        The variable of interest (e.g. Photos or Classifications)
618    """
619    direction_photos = pd.DataFrame()
620    direction_photos["category"] = direction_list
621    direction_counts = [0 for i in range(len(direction_photos))]
622    for mask in lc_df[bit_binary]:
623        for i in range(len(mask) - 1, -1, -1):
624            direction_counts[i] += int(mask[i])
625    direction_counts
626    direction_photos["count"] = [math.log10(value) for value in direction_counts]
627    direction_photos
628
629    plt.figure(figsize=(15, 6))
630    title = f"Land Cover -- {entry_type} Direction Frequency (Log Scale)"
631    plt.title(title)
632    plt.ylabel("Count (Log Scale)")
633    sns.barplot(data=direction_photos, x="category", y="count", color="lightblue")
634
635
636def diagnostic_plots(
637    lc_df,
638    up_url="lc_UpwardPhotoUrl",
639    down_url="lc_DownwardPhotoUrl",
640    north_url="lc_NorthPhotoUrl",
641    south_url="lc_SouthPhotoUrl",
642    east_url="lc_EastPhotoUrl",
643    west_url="lc_WestPhotoUrl",
644    photo_bit="lc_PhotoBitBinary",
645    north_classification="lc_NorthClassifications",
646    south_classification="lc_SouthClassifications",
647    east_classification="lc_EastClassifications",
648    west_classification="lc_WestClassifications",
649    classification_bit="lc_ClassificationBitBinary",
650):
651    """
652    Generates (but doesn't display) diagnostic plots to gain insight into the current data.
653
654    Plots:
655    - Valid Photo Count Distribution
656    - Photo Distribution by direction
657    - Classification Distribution by direction
658    - Photo Status Distribution
659    - Completeness Score Distribution
660    - Subcompleteness Score Distribution
661
662    Parameters
663    ----------
664    lc_df : pd.DataFrame
665        The DataFrame containing Flagged and Cleaned Land Cover Data.
666    """
667    plot_freq_bar(
668        lc_df, "Land Cover", "lc_PhotoCount", "Valid Photo Count", log_scale=True
669    )
670    direction_frequency(
671        lc_df,
672        [up_url, down_url, north_url, south_url, east_url, west_url],
673        photo_bit,
674        "Photo",
675    )
676    direction_frequency(
677        lc_df,
678        [
679            north_classification,
680            south_classification,
681            east_classification,
682            west_classification,
683        ],
684        classification_bit,
685        "Classification",
686    )
687    multiple_bar_graph(
688        lc_df,
689        "Land Cover",
690        ["lc_PhotoCount", "lc_RejectedCount", "lc_EmptyCount"],
691        "Photo Summary",
692        log_scale=True,
693    )
694
695    completeness_histogram(
696        lc_df, "Land Cover", "lc_CumulativeCompletenessScore", "Cumulative Completeness"
697    )
698    completeness_histogram(
699        lc_df, "Land Cover", "lc_SubCompletenessScore", "Sub Completeness"
700    )
701
702
703def qa_filter(
704    lc_df,
705    has_classification=False,
706    has_photo=False,
707    has_all_photos=False,
708    has_all_classifications=False,
709):
710    """
711    Can filter a cleaned and flagged mosquito habitat mapper DataFrame based on the following criteria:
712    - `Has Classification`: If the entry has atleast one direction classified
713    - `Has Photo` : If the entry has atleast one photo taken
714    - `Has All Photos` : If the entry has all photos taken (up, down, north, south, east, west)
715    - `Has All Classifications` : If the entry has all directions classified
716
717    Returns a copy of the DataFrame
718
719    Parameters
720    ----------
721    has_classification : bool, default=False
722        If True, only entries with atleast one classification will be included.
723    has_photo : bool, default=False
724        If True, only entries with atleast one photo will be included.
725    has_all_photos : bool, default=False
726        If True, only entries with all photos will be included.
727    has_all_classifications : bool, default=False
728        If True, only entries with all classifications will be included.
729
730    Returns
731    -------
732    pd.DataFrame
733        A DataFrame of the applied filters.
734    """
735
736    if has_classification and not has_all_classifications:
737        lc_df = lc_df[lc_df["lc_ClassificationBitDecimal"] > 0]
738    elif has_all_classifications:
739        lc_df = lc_df[lc_df["lc_ClassificationBitDecimal"] == 15]
740    if has_photo and not has_all_photos:
741        lc_df = lc_df[lc_df["lc_PhotoBitDecimal"] > 0]
742    elif has_all_photos:
743        lc_df = lc_df[lc_df["lc_PhotoBitDecimal"] == 63]
744
745    return lc_df
746
747
748def _accumulate_ties(classification_list):
749    classifications = list()
750    i = 0
751    while i < len(classification_list) - 1:
752        if classification_list[i][1] == classification_list[i + 1][1]:
753            classifications.append(classification_list[i][0])
754            classifications.append(classification_list[i + 1][0])
755            i += 1
756        else:
757            break
758
759    output = ", ".join([classification for classification in classifications])
760    if not output:
761        if len(classification_list) != 0:
762            output = classification_list[0][0]
763        else:
764            output = "NA"
765    # TODO replace w regex methods
766    return output, i + 1
767
768
769def _rank_direction(classification_dict, direction_classifications):
770    if pd.isna(direction_classifications):
771        return "NA", "NA"
772    classifications_list = []
773    classifications = direction_classifications.split(";")
774    for classification_data in classifications:
775        percent = extract_classification_percentage(classification_data)
776        classification = extract_classification_name(classification_data)
777        if classification in classification_dict:
778            classification_dict[classification] += percent
779        else:
780            classification_dict[classification] = percent
781        classifications_list.append((classification, percent))
782    classifications_list = sorted(
783        classifications_list, key=lambda x: x[1], reverse=True
784    )
785    if len(classifications_list) < 2:
786        return classifications_list[0][0], "NA"
787
788    primary_classification, i = _accumulate_ties(classifications_list)
789    secondary_classification, temp = _accumulate_ties(classifications_list[i:])
790
791    return primary_classification, secondary_classification
792
793
794def _rank_classifications(*args):
795    classification_dict = {}
796    rank_directions = [
797        classification
798        for arg in args
799        for classification in _rank_direction(classification_dict, arg)
800    ]
801    primary, secondary = ("NA", 0), ("NA", 0)
802    if classification_dict:
803        if len(classification_dict) < 2:
804            primary = (
805                list(classification_dict.keys())[0],
806                list(classification_dict.values())[0],
807            )
808        else:
809            sorted_classifications = sorted(
810                classification_dict.items(), key=lambda x: x[1], reverse=True
811            )
812            primary, i = _accumulate_ties(sorted_classifications)
813            primary = primary, sorted_classifications[0][1]
814            if i < len(sorted_classifications):
815                secondary, temp = _accumulate_ties(sorted_classifications[i:])
816                secondary = secondary, sorted_classifications[i][1]
817    return (
818        *rank_directions,
819        primary[0],
820        secondary[0],
821        primary[1] / len(args),
822        secondary[1] / len(args),
823    )
824
825
826def get_main_classifications(
827    lc_df,
828    north_classification="lc_NorthClassifications",
829    east_classification="lc_EastClassifications",
830    south_classification="lc_SouthClassifications",
831    west_classification="lc_WestClassifications",
832    north_primary="lc_NorthPrimary",
833    north_secondary="lc_NorthSecondary",
834    east_primary="lc_EastPrimary",
835    east_secondary="lc_EastSecondary",
836    south_primary="lc_SouthPrimary",
837    south_secondary="lc_SouthSecondary",
838    west_primary="lc_WestPrimary",
839    west_secondary="lc_WestSecondary",
840    primary_classification="lc_PrimaryClassification",
841    secondary_classification="lc_SecondaryClassification",
842    primary_percentage="lc_PrimaryPercentage",
843    secondary_percentage="lc_SecondaryPercentage",
844    inplace=False,
845):
846    if not inplace:
847        lc_df = lc_df.copy()
848    vectorized_rank = np.vectorize(_rank_classifications)
849    (
850        lc_df[north_primary],
851        lc_df[north_secondary],
852        lc_df[east_primary],
853        lc_df[east_secondary],
854        lc_df[south_primary],
855        lc_df[south_secondary],
856        lc_df[west_primary],
857        lc_df[west_secondary],
858        lc_df[primary_classification],
859        lc_df[secondary_classification],
860        lc_df[primary_percentage],
861        lc_df[secondary_percentage],
862    ) = vectorized_rank(
863        lc_df[north_classification].to_numpy(),
864        lc_df[east_classification].to_numpy(),
865        lc_df[south_classification].to_numpy(),
866        lc_df[west_classification].to_numpy(),
867    )
868
869    if not inplace:
870        return lc_df