iota_analytics_indexer/handlers/
mod.rs

1// Copyright (c) Mysten Labs, Inc.
2// Modifications Copyright (c) 2024 IOTA Stiftung
3// SPDX-License-Identifier: Apache-2.0
4
5use std::collections::{BTreeMap, BTreeSet};
6
7use anyhow::{Result, bail};
8use iota_data_ingestion_core::Worker;
9use iota_package_resolver::{PackageStore, Resolver};
10use iota_types::{
11    base_types::{ObjectID, StructTag, TypeTag},
12    effects::{TransactionEffects, TransactionEffectsAPI},
13    iota_sdk_types_conversions::struct_tag_core_to_sdk,
14    object::{Object, Owner, bounded_visitor::BoundedVisitor},
15    transaction::{SenderSignedData, TransactionDataAPI},
16};
17use move_core_types::annotated_value::{MoveStruct, MoveTypeLayout, MoveValue};
18
19use crate::{
20    FileType,
21    tables::{InputObjectKind, ObjectStatus, OwnerType},
22};
23
24pub mod checkpoint_handler;
25pub mod df_handler;
26pub mod event_handler;
27pub mod move_call_handler;
28pub mod object_handler;
29pub mod package_handler;
30pub mod transaction_handler;
31pub mod transaction_objects_handler;
32pub mod wrapped_object_handler;
33
34const WRAPPED_INDEXING_DISALLOW_LIST: [&str; 4] = [
35    "0x1::string::String",
36    "0x1::ascii::String",
37    "0x2::url::Url",
38    "0x2::object::ID",
39];
40
41#[async_trait::async_trait]
42pub trait AnalyticsHandler<S>: Worker<Message = (), Error = anyhow::Error> {
43    /// Read back rows which are ready to be persisted. This function
44    /// will be invoked by the analytics processor after every call to
45    /// process_checkpoint
46    async fn read(&self) -> Result<Vec<S>>;
47    /// Type of data being written by this processor i.e. checkpoint, object,
48    /// etc
49    fn file_type(&self) -> Result<FileType>;
50    fn name(&self) -> &str;
51}
52
53fn initial_shared_version(object: &Object) -> Option<u64> {
54    match object.owner {
55        Owner::Shared {
56            initial_shared_version,
57        } => Some(initial_shared_version.as_u64()),
58        _ => None,
59    }
60}
61
62fn get_owner_type(object: &Object) -> OwnerType {
63    match object.owner {
64        Owner::AddressOwner(_) => OwnerType::AddressOwner,
65        Owner::ObjectOwner(_) => OwnerType::ObjectOwner,
66        Owner::Shared { .. } => OwnerType::Shared,
67        Owner::Immutable => OwnerType::Immutable,
68    }
69}
70
71fn get_owner_address(object: &Object) -> Option<String> {
72    match object.owner {
73        Owner::AddressOwner(address) => Some(address.to_string()),
74        Owner::ObjectOwner(address) => Some(address.to_string()),
75        Owner::Shared { .. } => None,
76        Owner::Immutable => None,
77    }
78}
79
80// Helper class to track input object kind.
81// Build sets of object ids for input, shared input and gas coin objects as
82// defined in the transaction data.
83// Input objects include coins and shared.
84struct InputObjectTracker {
85    shared: BTreeSet<ObjectID>,
86    coins: BTreeSet<ObjectID>,
87    input: BTreeSet<ObjectID>,
88}
89
90impl InputObjectTracker {
91    fn new(txn: &SenderSignedData) -> Self {
92        let shared: BTreeSet<ObjectID> = txn
93            .shared_input_objects()
94            .into_iter()
95            .map(|shared_io| shared_io.id())
96            .collect();
97        let tx_data = txn.transaction_data();
98        let coins: BTreeSet<ObjectID> = tx_data
99            .gas()
100            .iter()
101            .map(|obj_ref| obj_ref.object_id)
102            .collect();
103        // All input objects (transaction + authenticators) are collected here, just
104        // like the shared objects previously.
105        let input: BTreeSet<ObjectID> = txn
106            .input_objects()
107            .expect("input objects must be valid")
108            .into_iter()
109            .map(|io_kind| io_kind.object_id())
110            .collect();
111        Self {
112            shared,
113            coins,
114            input,
115        }
116    }
117
118    fn get_input_object_kind(&self, object_id: &ObjectID) -> Option<InputObjectKind> {
119        if self.coins.contains(object_id) {
120            Some(InputObjectKind::GasCoin)
121        } else if self.shared.contains(object_id) {
122            Some(InputObjectKind::SharedInput)
123        } else if self.input.contains(object_id) {
124            Some(InputObjectKind::Input)
125        } else {
126            None
127        }
128    }
129}
130
131// Helper class to track object status.
132// Build sets of object ids for created, mutated and deleted objects as reported
133// in the transaction effects.
134struct ObjectStatusTracker {
135    created: BTreeSet<ObjectID>,
136    mutated: BTreeSet<ObjectID>,
137    deleted: BTreeSet<ObjectID>,
138}
139
140impl ObjectStatusTracker {
141    fn new(effects: &TransactionEffects) -> Self {
142        let created: BTreeSet<ObjectID> = effects
143            .created()
144            .iter()
145            .map(|(obj_ref, _)| obj_ref.object_id)
146            .collect();
147        let mutated: BTreeSet<ObjectID> = effects
148            .mutated()
149            .iter()
150            .chain(effects.unwrapped().iter())
151            .map(|(obj_ref, _)| obj_ref.object_id)
152            .collect();
153        let deleted: BTreeSet<ObjectID> = effects
154            .all_tombstones()
155            .into_iter()
156            .map(|(id, _)| id)
157            .collect();
158        Self {
159            created,
160            mutated,
161            deleted,
162        }
163    }
164
165    fn get_object_status(&self, object_id: &ObjectID) -> Option<ObjectStatus> {
166        if self.mutated.contains(object_id) {
167            Some(ObjectStatus::Mutated)
168        } else if self.deleted.contains(object_id) {
169            Some(ObjectStatus::Deleted)
170        } else if self.created.contains(object_id) {
171            Some(ObjectStatus::Created)
172        } else {
173            None
174        }
175    }
176}
177
178async fn get_move_struct<T: PackageStore>(
179    struct_tag: &StructTag,
180    contents: &[u8],
181    resolver: &Resolver<T>,
182) -> Result<MoveStruct> {
183    let move_struct = match resolver
184        .type_layout(TypeTag::Struct(Box::new(struct_tag.clone())))
185        .await?
186    {
187        MoveTypeLayout::Struct(move_struct_layout) => {
188            BoundedVisitor::deserialize_struct(contents, &move_struct_layout)
189        }
190        _ => bail!("object is not a move struct"),
191    }?;
192    Ok(move_struct)
193}
194
195#[derive(Debug, Default)]
196pub struct WrappedStruct {
197    object_id: Option<ObjectID>,
198    struct_tag: Option<StructTag>,
199}
200
201fn parse_struct(
202    path: &str,
203    move_struct: MoveStruct,
204    all_structs: &mut BTreeMap<String, WrappedStruct>,
205) {
206    let mut wrapped_struct = WrappedStruct {
207        struct_tag: Some(struct_tag_core_to_sdk(&move_struct.type_)),
208        ..Default::default()
209    };
210    for (k, v) in move_struct.fields {
211        parse_struct_field(&format!("{path}.{k}"), v, &mut wrapped_struct, all_structs);
212    }
213    all_structs.insert(path.to_string(), wrapped_struct);
214}
215
216fn parse_struct_field(
217    path: &str,
218    move_value: MoveValue,
219    curr_struct: &mut WrappedStruct,
220    all_structs: &mut BTreeMap<String, WrappedStruct>,
221) {
222    match move_value {
223        MoveValue::Struct(move_struct) => {
224            let values = move_struct
225                .fields
226                .iter()
227                .map(|(id, value)| (id.to_string(), value))
228                .collect::<BTreeMap<_, _>>();
229            let struct_name = format!(
230                "0x{}::{}::{}",
231                move_struct.type_.address.short_str_lossless(),
232                move_struct.type_.module,
233                move_struct.type_.name
234            );
235            if "0x2::object::UID" == struct_name {
236                if let Some(MoveValue::Struct(id_struct)) = values.get("id").cloned() {
237                    let id_values = id_struct
238                        .fields
239                        .iter()
240                        .map(|(id, value)| (id.to_string(), value))
241                        .collect::<BTreeMap<_, _>>();
242                    if let Some(MoveValue::Address(address) | MoveValue::Signer(address)) =
243                        id_values.get("bytes").cloned()
244                    {
245                        curr_struct.object_id = Some(ObjectID::new(address.into_bytes()))
246                    }
247                }
248            } else if "0x1::option::Option" == struct_name {
249                // Option in iota move is implemented as vector of size 1
250                if let Some(MoveValue::Vector(vec_values)) = values.get("vec").cloned() {
251                    if let Some(first_value) = vec_values.first() {
252                        parse_struct_field(
253                            &format!("{path}[0]"),
254                            first_value.clone(),
255                            curr_struct,
256                            all_structs,
257                        );
258                    }
259                }
260            } else if !WRAPPED_INDEXING_DISALLOW_LIST.contains(&&*struct_name) {
261                // Do not index most common struct types i.e. string, url, etc
262                parse_struct(path, move_struct, all_structs)
263            }
264        }
265        MoveValue::Variant(v) => {
266            for (k, field) in v.fields.iter() {
267                parse_struct_field(
268                    &format!("{path}.{k}"),
269                    field.clone(),
270                    curr_struct,
271                    all_structs,
272                );
273            }
274        }
275        MoveValue::Vector(fields) => {
276            for (index, field) in fields.iter().enumerate() {
277                parse_struct_field(
278                    &format!("{path}[{index}]"),
279                    field.clone(),
280                    curr_struct,
281                    all_structs,
282                );
283            }
284        }
285        _ => {}
286    }
287}
288
289#[cfg(test)]
290mod tests {
291    use std::{collections::BTreeMap, str::FromStr};
292
293    use iota_types::base_types::{ObjectID, StructTag};
294    use move_core_types::{
295        account_address::AccountAddress,
296        annotated_value::{MoveStruct, MoveValue, MoveVariant},
297        identifier::Identifier,
298        language_storage::StructTag as MoveStructTag,
299    };
300
301    use crate::handlers::parse_struct;
302
303    #[tokio::test]
304    async fn test_wrapped_object_parsing() -> anyhow::Result<()> {
305        let uid_field = MoveValue::Struct(MoveStruct {
306            type_: MoveStructTag::from_str("0x2::object::UID")?,
307            fields: vec![(
308                Identifier::from_str("id")?,
309                MoveValue::Struct(MoveStruct {
310                    type_: MoveStructTag::from_str("0x2::object::ID")?,
311                    fields: vec![(
312                        Identifier::from_str("bytes")?,
313                        MoveValue::Signer(AccountAddress::from_hex_literal("0x300")?),
314                    )],
315                }),
316            )],
317        });
318        let balance_field = MoveValue::Struct(MoveStruct {
319            type_: MoveStructTag::from_str("0x2::balance::Balance")?,
320            fields: vec![(Identifier::from_str("value")?, MoveValue::U32(10))],
321        });
322        let move_struct = MoveStruct {
323            type_: MoveStructTag::from_str("0x2::test::Test")?,
324            fields: vec![
325                (Identifier::from_str("id")?, uid_field),
326                (Identifier::from_str("principal")?, balance_field),
327            ],
328        };
329        let mut all_structs = BTreeMap::new();
330        parse_struct("$", move_struct, &mut all_structs);
331        assert_eq!(
332            all_structs.get("$").unwrap().object_id,
333            Some(ObjectID::from_short_hex("0x300")?)
334        );
335        assert_eq!(
336            all_structs.get("$.principal").unwrap().struct_tag,
337            Some(StructTag::from_str("0x2::balance::Balance")?)
338        );
339        Ok(())
340    }
341
342    #[tokio::test]
343    async fn test_wrapped_object_parsing_within_enum() -> anyhow::Result<()> {
344        let uid_field = MoveValue::Struct(MoveStruct {
345            type_: MoveStructTag::from_str("0x2::object::UID")?,
346            fields: vec![(
347                Identifier::from_str("id")?,
348                MoveValue::Struct(MoveStruct {
349                    type_: MoveStructTag::from_str("0x2::object::ID")?,
350                    fields: vec![(
351                        Identifier::from_str("bytes")?,
352                        MoveValue::Signer(AccountAddress::from_hex_literal("0x300")?),
353                    )],
354                }),
355            )],
356        });
357        let balance_field = MoveValue::Struct(MoveStruct {
358            type_: MoveStructTag::from_str("0x2::balance::Balance")?,
359            fields: vec![(Identifier::from_str("value")?, MoveValue::U32(10))],
360        });
361        let move_enum = MoveVariant {
362            type_: MoveStructTag::from_str("0x2::test::TestEnum")?,
363            variant_name: Identifier::from_str("TestVariant")?,
364            tag: 0,
365            fields: vec![
366                (Identifier::from_str("field0")?, MoveValue::U64(10)),
367                (Identifier::from_str("principal")?, balance_field),
368            ],
369        };
370        let move_struct = MoveStruct {
371            type_: MoveStructTag::from_str("0x2::test::Test")?,
372            fields: vec![
373                (Identifier::from_str("id")?, uid_field),
374                (
375                    Identifier::from_str("enum_field")?,
376                    MoveValue::Variant(move_enum),
377                ),
378            ],
379        };
380        let mut all_structs = BTreeMap::new();
381        parse_struct("$", move_struct, &mut all_structs);
382        assert_eq!(
383            all_structs.get("$").unwrap().object_id,
384            Some(ObjectID::from_short_hex("0x300")?)
385        );
386        assert_eq!(
387            all_structs
388                .get("$.enum_field.principal")
389                .unwrap()
390                .struct_tag,
391            Some(StructTag::from_str("0x2::balance::Balance")?)
392        );
393        Ok(())
394    }
395}
iota_analytics_indexer/handlers/mod.rs

iota_analytics_indexer/handlers/
mod.rs
