Skip to content

C
Clean
Commit 265b180a authored by maliang's avatar maliang
Browse files
Options

clean

parent 18a22379
Hide whitespace changes
Inline Side-by-side
Showing with 2626 additions and 0 deletions
clean/.idea/compiler.xml 0 → 100644
View file @ 265b180a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="clean" />
</profile>
</annotationProcessing>
</component>
</project>
\ No newline at end of file
clean/.idea/misc.xml 0 → 100644
View file @ 265b180a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
clean/.idea/scala_settings.xml 0 → 100644
View file @ 265b180a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaProjectSettings">
<option name="customScalatestSyntaxHighlighting" value="true" />
</component>
</project>
\ No newline at end of file
clean/.idea/workspace.xml 0 → 100644
View file @ 265b180a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="a7b5baec-c31b-43f5-8568-411c09ea07eb" name="Default" comment="" />
<ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="clean_api.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6707">
<caret line="353" column="31" selection-start-line="353" selection-start-column="31" selection-end-line="353" selection-end-column="31" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="clean_api_android.sh" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api_android.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="304">
<caret line="64" column="60" selection-start-line="64" selection-start-column="60" selection-end-line="64" selection-end-column="60" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-304">
<caret line="14" column="12" selection-start-line="14" selection-start-column="16" selection-end-line="14" selection-end-column="39" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="CleanUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/CleanUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="237">
<caret line="36" column="40" selection-start-line="36" selection-start-column="40" selection-end-line="36" selection-end-column="40" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="CleanApi.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanApi.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="27" column="27" selection-start-line="27" selection-start-column="27" selection-end-line="27" selection-end-column="27" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="CleanDriver.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanDriver.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-114">
<caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Test.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/test/java/com/novel/test/Test.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="266">
<caret line="14" column="51" selection-start-line="14" selection-start-column="51" selection-end-line="14" selection-end-column="51" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Class" />
</list>
</option>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>referaction_detail</find>
<find>logversion</find>
<find>detail_area</find>
<find>requrlstr</find>
<find>eval</find>
<find>reqParam</find>
<find>refer_com</find>
<find>referaction</find>
<find>querymessage</find>
<find>consumptiontype</find>
<find>ua</find>
<find>parameter</find>
<find>platform</find>
<find>day</find>
</findStrings>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/resources/log4j.properties" />
<option value="$PROJECT_DIR$/src/main/java/com/novel/clean/util/IPZone.java" />
<option value="$PROJECT_DIR$/src/main/resources/clean_api.py" />
<option value="$PROJECT_DIR$/src/main/java/com/novel/clean/util/QQWryIPUtil.java" />
<option value="$PROJECT_DIR$/src/main/resources/api.json" />
<option value="$PROJECT_DIR$/src/test/java/com/novel/test/Test.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/java/com/novel/clean/CleanDriver.java" />
<option value="$PROJECT_DIR$/src/main/java/com/novel/clean/util/CleanUtil.java" />
<option value="$PROJECT_DIR$/src/main/java/com/novel/clean/CleanApi.java" />
</list>
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="MavenImportPreferences">
<option name="generalSettings">
<MavenGeneralSettings>
<option name="localRepository" value="F:\repository\repository" />
<option name="mavenHome" value="H:/apache-maven-3.2.2" />
<option name="userSettingsFile" value="F:\repository\settings.xml" />
</MavenGeneralSettings>
</option>
</component>
<component name="MavenProjectNavigator">
<treeState>
<expand>
<path>
<item name="" type="16c1761:MavenProjectsStructure$RootNode" />
<item name="clean" type="9519ce18:MavenProjectsStructure$ProjectNode" />
</path>
<path>
<item name="" type="16c1761:MavenProjectsStructure$RootNode" />
<item name="clean" type="9519ce18:MavenProjectsStructure$ProjectNode" />
<item name="Lifecycle" type="58874e2:MavenProjectsStructure$LifecycleNode" />
</path>
</expand>
<select />
</treeState>
</component>
<component name="NodePackageJsonFileManager">
<packageJsonPaths />
</component>
<component name="ProjectFrameBounds" extendedState="7">
<option name="x" value="95" />
<option name="y" value="35" />
<option name="width" value="1750" />
<option name="height" value="970" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="resources" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="test" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="test" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="clean" type="b2602c69:ProjectViewProjectNode" />
<item name="clean" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="test" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="test" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
<pane id="PackagesPane" />
<pane id="AndroidView" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="project.structure.last.edited" value="Project" />
<property name="project.structure.proportion" value="0.0" />
<property name="project.structure.side.proportion" value="0.0" />
<property name="settings.editor.selected.configurable" value="org.jetbrains.plugins.scala.settings.ScalaProjectSettingsConfigurable" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="H:\look novel\clean\src\main\java\com\novel\clean\util" />
<recent name="H:\look novel\clean\src\main\resources" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Application.CleanDriver">
<configuration name="CleanDriver" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.novel.clean.CleanDriver" />
<module name="clean" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.novel.clean.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration name="QQWryIPUtil" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.novel.clean.util.QQWryIPUtil" />
<module name="clean" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.novel.clean.util.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration name="Test" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.novel.test.Test" />
<module name="clean" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.novel.test.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<patterns />
</configuration>
<configuration default="true" type="TestNG" factoryName="TestNG">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="SUITE_NAME" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="GROUP_NAME" />
<option name="TEST_OBJECT" value="CLASS" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="OUTPUT_DIRECTORY" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<option name="USE_DEFAULT_REPORTERS" value="false" />
<option name="PROPERTIES_FILE" />
<properties />
<listeners />
</configuration>
<list>
<item itemvalue="Application.QQWryIPUtil" />
<item itemvalue="Application.Test" />
<item itemvalue="Application.CleanDriver" />
</list>
<recent_temporary>
<list>
<item itemvalue="Application.CleanDriver" />
<item itemvalue="Application.Test" />
<item itemvalue="Application.QQWryIPUtil" />
</list>
</recent_temporary>
</component>
<component name="ScalaProjectSettings">
<option name="customScalatestSyntaxHighlighting" value="true" />
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="a7b5baec-c31b-43f5-8568-411c09ea07eb" name="Default" comment="" />
<created>1570846488582</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1570846488582</updated>
<workItem from="1570846492227" duration="948000" />
<workItem from="1570847667770" duration="29159000" />
<workItem from="1570884506094" duration="1153000" />
<workItem from="1570957983362" duration="4801000" />
<workItem from="1570966870339" duration="6718000" />
<workItem from="1571035319725" duration="6227000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="49006000" />
</component>
<component name="ToolWindowManager">
<frame x="-7" y="-7" width="1550" height="838" extended-state="7" />
<editor active="true" />
<layout>
<window_info anchor="right" id="Palette" order="3" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Messages" order="7" />
<window_info anchor="right" id="Palette&#9;" order="3" />
<window_info id="Image Layers" order="2" />
<window_info anchor="bottom" id="Java Enterprise" order="7" />
<window_info anchor="right" id="Capture Analysis" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" side_tool="true" />
<window_info anchor="right" id="Maven Projects" order="3" weight="0.3297587" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info active="true" anchor="bottom" id="Run" order="2" visible="true" weight="0.059322033" />
<window_info anchor="bottom" id="Terminal" order="7" />
<window_info id="Capture Tool" order="2" />
<window_info id="Designer" order="2" />
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.26675603" />
<window_info anchor="right" id="Database" order="3" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" order="2" />
<window_info anchor="right" id="Theme Preview" order="3" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Find" order="1" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6707">
<caret line="353" column="31" selection-start-line="353" selection-start-column="31" selection-end-line="353" selection-end-column="31" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api_android.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1216">
<caret line="64" column="60" selection-start-line="64" selection-start-column="60" selection-end-line="64" selection-end-column="60" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="722">
<caret line="38" column="12" selection-start-line="38" selection-start-column="12" selection-end-line="38" selection-end-column="35" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/CleanUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="817">
<caret line="47" column="61" selection-start-line="47" selection-start-column="61" selection-end-line="47" selection-end-column="61" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanApi.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2964">
<caret line="163" column="83" selection-start-line="163" selection-start-column="77" selection-end-line="163" selection-end-column="83" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/novel/test/Test.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="14" column="8" selection-start-line="14" selection-start-column="8" selection-end-line="14" selection-end-column="51" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanDriver.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="114">
<caret line="12" column="10" selection-start-line="12" selection-start-column="10" selection-end-line="12" selection-end-column="10" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6707">
<caret line="353" column="31" selection-start-line="353" selection-start-column="31" selection-end-line="353" selection-end-column="31" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api_android.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1216">
<caret line="64" column="60" selection-start-line="64" selection-start-column="60" selection-end-line="64" selection-end-column="60" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/CleanUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1102">
<caret line="62" column="43" selection-start-line="62" selection-start-column="43" selection-end-line="62" selection-end-column="43" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanApi.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1691">
<caret line="89" column="1" selection-start-line="89" selection-start-column="1" selection-end-line="89" selection-end-column="1" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanDriver.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="95">
<caret line="5" column="31" selection-start-line="5" selection-start-column="31" selection-end-line="5" selection-end-column="31" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/novel/test/Test.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="266">
<caret line="14" column="8" selection-start-line="14" selection-start-column="8" selection-end-line="14" selection-end-column="51" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="380">
<caret line="20" selection-start-line="20" selection-end-line="20" selection-end-column="35" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6707">
<caret line="353" column="31" selection-start-line="353" selection-start-column="31" selection-end-line="353" selection-end-column="31" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api_android.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1216">
<caret line="64" column="60" selection-start-line="64" selection-start-column="60" selection-end-line="64" selection-end-column="60" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="722">
<caret line="38" column="37" selection-start-line="38" selection-start-column="37" selection-end-line="38" selection-end-column="37" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/CleanUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1102">
<caret line="62" column="43" selection-start-line="62" selection-start-column="43" selection-end-line="62" selection-end-column="43" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanApi.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="57">
<caret line="10" column="21" selection-start-line="10" selection-start-column="13" selection-end-line="10" selection-end-column="21" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/novel/test/Test.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="14" column="8" selection-start-line="14" selection-start-column="8" selection-end-line="14" selection-end-column="51" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanDriver.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="323">
<caret line="24" column="10" lean-forward="true" selection-start-line="24" selection-start-column="10" selection-end-line="24" selection-end-column="10" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6707">
<caret line="353" column="31" lean-forward="true" selection-start-line="353" selection-start-column="31" selection-end-line="353" selection-end-column="31" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api_android.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1216">
<caret line="64" column="60" selection-start-line="64" selection-start-column="60" selection-end-line="64" selection-end-column="60" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="722">
<caret line="38" column="37" selection-start-line="38" selection-start-column="37" selection-end-line="38" selection-end-column="37" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/CleanUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1178">
<caret line="62" column="43" selection-start-line="62" selection-start-column="43" selection-end-line="62" selection-end-column="43" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/novel/test/Test.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="14" column="8" lean-forward="true" selection-start-line="14" selection-start-column="8" selection-end-line="14" selection-end-column="51" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanApi.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1957">
<caret line="110" column="46" selection-start-line="110" selection-start-column="46" selection-end-line="110" selection-end-column="46" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="684">
<caret line="36" selection-start-line="36" selection-end-line="36" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/log4j.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="57">
<caret line="3" column="21" selection-start-line="3" selection-start-column="21" selection-end-line="3" selection-end-column="21" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.sh">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="jar://H:/MyWork/Program/jdk1.8.0_202/src.zip!/java/util/Map.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="226">
<caret line="324" column="11" selection-start-line="324" selection-start-column="11" selection-end-line="324" selection-end-column="11" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/IPZone.java">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/api.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="19">
<caret line="1" lean-forward="true" selection-start-line="1" selection-end-line="1" />
</state>
</provider>
</entry>
<entry file="jar://H:/MyWork/Program/jdk1.8.0_202/src.zip!/java/lang/String.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-757">
<caret line="2328" column="20" selection-start-line="2328" selection-start-column="20" selection-end-line="2328" selection-end-column="20" />
<folding>
<element signature="e#94598#94599#0" expanded="true" />
<element signature="e#94636#94637#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-common/2.7.2/hadoop-mapreduce-client-common-2.7.2-sources.jar!/org/apache/hadoop/mapred/LocalJobRunner.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="305">
<caret line="459" column="46" selection-start-line="459" selection-start-column="37" selection-end-line="459" selection-end-column="46" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6707">
<caret line="353" column="31" selection-start-line="353" selection-start-column="31" selection-end-line="353" selection-end-column="31" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/novel/test/Test.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="266">
<caret line="14" column="51" selection-start-line="14" selection-start-column="51" selection-end-line="14" selection-end-column="51" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-304">
<caret line="14" column="12" selection-start-line="14" selection-start-column="16" selection-end-line="14" selection-end-column="39" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/QQWryIPUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="128">
<caret line="246" column="37" selection-start-line="246" selection-start-column="37" selection-end-line="246" selection-end-column="37" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/clean_api_android.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="304">
<caret line="64" column="60" selection-start-line="64" selection-start-column="60" selection-end-line="64" selection-end-column="60" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/util/CleanUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="237">
<caret line="36" column="40" selection-start-line="36" selection-start-column="40" selection-end-line="36" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanDriver.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-114">
<caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/novel/clean/CleanApi.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="27" column="27" selection-start-line="27" selection-start-column="27" selection-end-line="27" selection-end-column="27" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>
\ No newline at end of file
clean/clean.iml 0 → 100644
View file @ 265b180a
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
clean/pom.xml 0 → 100644
View file @ 265b180a
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.novel.clean</groupId>
<artifactId>clean</artifactId>
<version>1.0</version>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>RELEASE</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.8.2</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.2</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.2</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.2</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
clean/src/main/java/com/novel/clean/CleanApi.java 0 → 100644
View file @ 265b180a
package com.novel.clean;
import java.io.IOException;
import java.util.Map;
import com.novel.clean.util.CleanUtil;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class CleanApi extends Mapper<LongWritable, Text, Text, NullWritable>{
private Text line = new Text();
private CleanUtil cleanUtil=new CleanUtil();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] list = value.toString().split("] \\[");
String logversion = "-999";
String projectname = "-999";
String serverip = "-999"; //服务器ip
String reqbegintime = "-999"; //请求时间
String reqendtime = "-999";; //响应时间
String reqspendtime = "-999"; //时间差
String reqip = "-999"; //客户端ip
String requrlstr = "-999";//请求地址
String refer = "-999";
String method = "-999";;
String httpcode = "-999";
String userid = "-999";
String sessionid = "-999";
String platformid = "-999";
String appversion = "-999";
String channelid = "-999";
String bookrecommend= "-999";
String payType = "-999";
String paynum = "-999";
String rechargetype = "-999";
String payamount = "-999";
String ssid = "-999";
String isnewuser = "-999";
String ischannelnewuser = "-999";
String readwordstype = "-999";
String readwords = "-999";
String mobile = "-999";
String uaplatform = "-999";
String uastat = "-999";
String apicode = "-999";
String xClient = "-999";
String ua = "-999";
String reqParam = "-999";
String processParam = "-999";
try {
logversion = list[0].split("\\[")[1];
projectname = list[1];
serverip = list[2]; //服务器ip
reqbegintime = list[3]; //请求时间
reqendtime = list[4]; //响应时间
reqspendtime = list[5]; //时间差
reqip = list[6]; //客户端ip
requrlstr = list[7];//请求地址
refer = list[8];
method = list[9];
httpcode = list[10];
userid = list[11];
sessionid = list[12];
platformid = list[13];
appversion = list[14];
channelid = list[15];
bookrecommend= list[16];
payType = list[17];
paynum = list[18];
rechargetype = list[19];
payamount = list[20];
ssid = list[21];
isnewuser = list[22];
ischannelnewuser = list[23];
readwordstype = list[24];
readwords = list[25];
mobile = list[26];
uaplatform = list[27];
uastat = list[28];
apicode = list[29];
xClient = list[30];
ua = list[31];
reqParam = list[32];
processParam = list[33];
}catch (Exception e){
}
//过滤下载日志
if((requrlstr.contains("/api/download/")) || (requrlstr.contains("/api/other/jrttBack")) || (requrlstr.contains("/kuaishou/notify")) || (requrlstr.contains("/api/getPublicKey")) || ua.contains("JianKongBao") )
return;
else if(requrlstr.contains("type=bbs")){
if (platformid == "1")
platformid="100";
else if (platformid == "2")
platformid="200";
}
//解析国家和地区
Map<String, String> ips = cleanUtil.getIp(reqip);
String country = ips.get("country");
String detail_area = ips.get("detail_area");
String province=ips.get("province");
String city=ips.get("city");
//解析action和action_detail
Map<String, String> actions = cleanUtil.getAction(requrlstr);
String action=actions.get("action");
String action_detail=actions.get("action_detail");
//解析bookid和chapternum
Map<String, String> book=cleanUtil.getBook(reqParam);
String bookid=book.get("bookId");
String chapternum=book.get("chapterNum");
//暂时用不到的三个属性
String referurl="-999";
String refer_com="-999";
String referaction="-999";
//切分Xclient取出相应的数据
String screensize="-999";
String type="-999";
String imsi="-999";
String version ="-999";
try {
screensize = xClient.split("ss=")[1].split(";")[0];
type = xClient.split("pm=")[1].split(";")[0];
imsi = xClient.split("imsi=")[1].split(";")[0];
version = xClient.split("version=")[1].split(";")[0];
}catch (Exception e){
}
String appversion2="-999";
if (version !=null && version.indexOf(".")>0 ){
int first_index = version.indexOf('.') + 1;
int last_index = version.lastIndexOf('.');
appversion2 = version.substring(first_index,last_index);
}
String mac ="-999";
String device_id ="-999";
String imei ="-999";
String idfa ="-999";
try {
mac = xClient.split("mac=")[1].split(";")[0];
device_id = xClient.split("dID=")[1].split(";")[0];
imei = xClient.split("imei=")[1].split(";")[0];
idfa = xClient.split("idfa=")[1].split(";")[0];
}catch (Exception e){
}
//暂时用不到的几个属性
String endbrand = "-999";
String browserbrand = "-999";
String browserversion = "-999";
String primaryurlpath = "-999";
String others = "-999";
String cellid = "-999";
String projectType="-999";
//解析操作系统版本
String os_version="-999";
String platform="api";
try {
os_version = ua.split(";")[1].split(" ")[2];
platform=ua.split(";")[1].split(" ")[1];
}catch (Exception e){
}
//日志时间
String day=reqbegintime.split(" ")[0];
line.set(logversion+"\t"+projectname+"\t"+serverip+"\t"+reqbegintime+"\t"+reqendtime+"\t"+reqspendtime+"\t"+reqip+"\t"+country
+"\t"+detail_area+"\t"+province+"\t"+city+"\t"+requrlstr+"\t"+action+"\t"+action_detail+"\t"+bookid+"\t"+chapternum+"\t"+referurl+"\t"+refer_com+"\t"+
referaction+"\t"+method+"\t"+httpcode+"\t"+userid+"\t"+sessionid+"\t"+platformid+"\t"+appversion+"\t"+channelid+"\t"+
bookrecommend+"\t"+payType+"\t"+paynum+"\t"+rechargetype+"\t"+payamount+"\t"+ssid+"\t"+isnewuser+"\t"+ischannelnewuser+"\t"+
readwordstype+"\t"+readwords+"\t"+mobile+"\t"+uaplatform+"\t"+uastat+"\t"+apicode+"\t"+imei+"\t"+os_version+"\t"+screensize
+"\t"+type+"\t"+imsi+"\t"+cellid+"\t"+version+"\t"+device_id+"\t"+mac+"\t"+endbrand+"\t"+browserbrand+"\t"+browserversion+"\t"+
ua+"\t"+primaryurlpath+"\t"+reqParam+"\t"+others+"\t"+processParam+"\t"+projectType+"\t"+idfa+"\t"+appversion2+"\t"+platform+"\t"+day);
context.write(line, NullWritable.get());
}
}
clean/src/main/java/com/novel/clean/CleanDriver.java 0 → 100644
View file @ 265b180a
package com.novel.clean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class CleanDriver {
public static void main(String[] args) throws Exception{
args = new String[]{"e:/input", "e:/output"};
// 1 获取job信息
Configuration configuration = new Configuration();
Job job = Job.getInstance(configuration);
//2 设置加载jar包路径
job.setJarByClass(CleanDriver.class);
//3 关联map
job.setMapperClass(CleanApi.class);
//4 设置最终输出数据类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
//5 设置输入输出路径
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//6 设置reduceTask数量为0
job.setNumReduceTasks(0);
//7 提交
boolean result = job.waitForCompletion(true);
System.exit(result ? 0 : 1);
}
}
clean/src/main/java/com/novel/clean/util/CleanUtil.java 0 → 100644
View file @ 265b180a
package com.novel.clean.util;
import com.alibaba.fastjson.JSONObject;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class CleanUtil {
public static QQWryIPUtil qqWryIPUtil;
public static IPZone ipZone;
static{
try {
qqWryIPUtil = new QQWryIPUtil();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* This method is used for incoming IP to resolve provinces and service providers
* 此方法用于传入ip解析出省份和服务商
* @author maliang
* */
public Map<String,String> getIp(String ip){
Map<String,String> analysis=new HashMap<>();
try {
ipZone= qqWryIPUtil.findIP(ip);
Map<String, String> zone = QQWryIPUtil.addressResolution(ipZone.getMainInfo());
analysis.put("country",ipZone.getMainInfo());
analysis.put("detail_area",ipZone.getSubInfo());
analysis.put("province",zone.get("province") ==""?"-999":zone.get("province"));
analysis.put("city",zone.get("city") ==""?"-999":zone.get("city"));
}catch (Exception e){
analysis.put("country","-999");
analysis.put("detail_area","-999");
analysis.put("province","-999");
analysis.put("city","-999");
}
return analysis;
}
/**
* This method resolves action and action_detail fields by requesting addresses
*这个方法可以通过请求地址把action和action_detail字段解析出来
* @author maliang
* */
public Map<String,String> getAction(String requrlstr){
Map<String,String> analysis=new HashMap<>();
analysis.put("action","-999");
analysis.put("action_detail","-999");
String[] split = requrlstr.split("/");
try {
analysis.put("action",split[2]+"_"+split[3]);
analysis.put("action_detail",split[2]+"_"+split[3]);
}catch (Exception e){
}
return analysis;
}
/**
* This method can parse the bookid and chapternum fields clicked by users in reqParam
*这个方法可以把reqParam里用户点击的bookid和chapternum字段解析出来
* @author maliang
* */
public Map<String,String> getBook(String reqParam){
Map<String,String> analysis=new HashMap<>();
try{
JSONObject jsonObject = JSONObject.parseObject(reqParam);
analysis.put("bookId",(jsonObject.getString("bookId")==null)?"-999":jsonObject.getString("bookId"));
analysis.put("chapterNum",jsonObject.getString("chapterNum")==null?"-999":jsonObject.getString("chapterNum"));
}catch (Exception e){
}
return analysis;
}
}
clean/src/main/java/com/novel/clean/util/IPZone.java 0 → 100644
View file @ 265b180a
package com.novel.clean.util;
/**
* @author: xsj
* @date:2019/8/7
*/
public class IPZone {
private final String ip;
private String mainInfo = "";
private String subInfo = "";
public IPZone(String ip) {
this.ip = ip;
}
public String getIp() {
return this.ip;
}
public String getMainInfo() {
return this.mainInfo;
}
public String getSubInfo() {
return this.subInfo;
}
public void setMainInfo(String info) {
this.mainInfo = info;
}
public void setSubInfo(String info) {
this.subInfo = info;
}
public String toString() {
return this.mainInfo + this.subInfo;
}
}
clean/src/main/java/com/novel/clean/util/QQWryIPUtil.java 0 → 100644
View file @ 265b180a
package com.novel.clean.util;
import com.novel.clean.util.IPZone;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author: xsj
* @date:2019/8/7
*/
public class QQWryIPUtil {
public static final Pattern IPZonePattern = Pattern.compile("(?<province>[^省]+自治区|.*?省|.*?行政区|.*?市)(?<city>[^市]+自治州|.*?地区|.*?行政单位|.+盟|市辖区|.*?市|.*?县)(?<county>[^县]+县|.+区|.+市|.+旗|.+海域|.+岛)?(?<town>[^区]+区|.+镇)?(?<village>.*)");
private static class QIndex {
public final long minIP;
public final long maxIP;
public final int recordOffset;
public QIndex(final long minIP, final long maxIP, final int recordOffset) {
this.minIP = minIP;
this.maxIP = maxIP;
this.recordOffset = recordOffset;
}
}
private static class QString {
public final String string;
/** length including the \0 end byte */
public final int length;
public QString(final String string, final int length) {
this.string = string;
this.length = length;
}
}
private static final int INDEX_RECORD_LENGTH = 7;
private static final byte REDIRECT_MODE_1 = 0x01;
private static final byte REDIRECT_MODE_2 = 0x02;
private static final byte STRING_END = '\0';
private final byte[] data;
private final long indexHead;
private final long indexTail;
/**
* Create QQWry by loading qqwry.dat from classpath.
*
* @throws IOException
* if encounter error while reading qqwry.dat
*/
public QQWryIPUtil() throws IOException {
ClassLoader classLoader = QQWryIPUtil.class.getClassLoader();
final InputStream in = classLoader.getResourceAsStream("qqwry.dat");
final ByteArrayOutputStream out = new ByteArrayOutputStream(10 * 1024 * 1024); // 10MB
final byte[] buffer = new byte[4096];
while (true) {
final int r = in.read(buffer);
if (r == -1) {
break;
}
out.write(buffer, 0, r);
}
data = out.toByteArray();
indexHead = readLong32(0);
indexTail = readLong32(4);
}
/**
* Create QQWry with provided qqwry.dat data.
*
* @param data
* fully read data from a qqwry.dat file.
*/
public QQWryIPUtil(final byte[] data) {
this.data = data;
indexHead = readLong32(0);
indexTail = readLong32(4);
}
/**
* Create QQWry from a path to qqwry.dat file.
*
* @param file
* path to qqwry.dat
* @throws IOException
* if encounter error while reading from the given file.
*/
public QQWryIPUtil(final Path file) throws IOException {
this(Files.readAllBytes(file));
}
public IPZone findIP(final String ip) {
final long ipNum = toNumericIP(ip);
final QIndex idx = searchIndex(ipNum);
if (idx == null) {
return new IPZone(ip);
}
return readIP(ip, idx);
}
private long getMiddleOffset(final long begin, final long end) {
long records = (end - begin) / INDEX_RECORD_LENGTH;
records >>= 1;
if (records == 0) {
records = 1;
}
return begin + (records * INDEX_RECORD_LENGTH);
}
private QIndex readIndex(final int offset) {
final long min = readLong32(offset);
final int record = readInt24(offset + 4);
final long max = readLong32(record);
return new QIndex(min, max, record);
}
private int readInt24(final int offset) {
int v = data[offset] & 0xFF;
v |= ((data[offset + 1] << 8) & 0xFF00);
v |= ((data[offset + 2] << 16) & 0xFF0000);
return v;
}
private IPZone readIP(final String ip, final QIndex idx) {
final int pos = idx.recordOffset + 4; // skip ip
final byte mode = data[pos];
final IPZone z = new IPZone(ip);
if (mode == REDIRECT_MODE_1) {
final int offset = readInt24(pos + 1);
if (data[offset] == REDIRECT_MODE_2) {
readMode2(z, offset);
} else {
final QString mainInfo = readString(offset);
final String subInfo = readSubInfo(offset + mainInfo.length);
z.setMainInfo(mainInfo.string);
z.setSubInfo(subInfo);
}
} else if (mode == REDIRECT_MODE_2) {
readMode2(z, pos);
} else {
final QString mainInfo = readString(pos);
final String subInfo = readSubInfo(pos + mainInfo.length);
z.setMainInfo(mainInfo.string);
z.setSubInfo(subInfo);
}
return z;
}
private long readLong32(final int offset) {
long v = data[offset] & 0xFFL;
v |= (data[offset + 1] << 8L) & 0xFF00L;
v |= ((data[offset + 2] << 16L) & 0xFF0000L);
v |= ((data[offset + 3] << 24L) & 0xFF000000L);
return v;
}
private void readMode2(final IPZone z, final int offset) {
final int mainInfoOffset = readInt24(offset + 1);
final String main = readString(mainInfoOffset).string;
final String sub = readSubInfo(offset + 4);
z.setMainInfo(main);
z.setSubInfo(sub);
}
private QString readString(final int offset) {
int i = 0;
final byte[] buf = new byte[128];
for (;; i++) {
final byte b = data[offset + i];
if (STRING_END == b) {
break;
}
buf[i] = b;
}
try {
return new QString(new String(buf, 0, i, "GB18030"), i + 1);
} catch (final UnsupportedEncodingException e) {
return new QString("", 0);
}
}
private String readSubInfo(final int offset) {
final byte b = data[offset];
if ((b == REDIRECT_MODE_1) || (b == REDIRECT_MODE_2)) {
final int areaOffset = readInt24(offset + 1);
if (areaOffset == 0) {
return "";
} else {
return readString(areaOffset).string;
}
} else {
return readString(offset).string;
}
}
private QIndex searchIndex(final long ip) {
long head = indexHead;
long tail = indexTail;
while (tail > head) {
final long cur = getMiddleOffset(head, tail);
final QIndex idx = readIndex((int) cur);
if ((ip >= idx.minIP) && (ip <= idx.maxIP)) {
return idx;
}
if ((cur == head) || (cur == tail)) {
return idx;
}
if (ip < idx.minIP) {
tail = cur;
} else if (ip > idx.maxIP) {
head = cur;
} else {
return idx;
}
}
return null;
}
private long toNumericIP(final String s) {
final String[] parts = s.split("\\.");
if (parts.length != 4) {
throw new IllegalArgumentException("ip=" + s);
}
long n = Long.parseLong(parts[0]) << 24L;
n += Long.parseLong(parts[1]) << 16L;
n += Long.parseLong(parts[2]) << 8L;
n += Long.parseLong(parts[3]);
return n;
}
/**
* 解析地址
* @author lin
* @param address
* @return
*/
public static Map<String,String> addressResolution(String address){
Matcher m= IPZonePattern.matcher(address);
String province,city,county,town,village;
Map<String,String> row=null;
if (m.find()){
row= new LinkedHashMap<>();
province=m.group("province");
row.put("province", province==null?"":province.trim());
city=m.group("city");
row.put("city", city==null?"":city.trim());
county=m.group("county");
row.put("county", county==null?"":county.trim());
town=m.group("town");
row.put("town", town==null?"":town.trim());
village=m.group("village");
row.put("village", village==null?"":village.trim());
}
return row;
}
public static void main(String[] args) {
try {
QQWryIPUtil qqWryIPUtil = new QQWryIPUtil();
// QQWryIPUtil qqWryIPUtil = new QQWryIPUtil(Paths.get("/Users/wangkun/Applications/qqwry.dat"));
IPZone ip = qqWryIPUtil.findIP("117.136.45.68");
// IPZone ip = qqWryIPUtil.findIP("192.168.31.13");
System.out.println(ip.getMainInfo());
System.out.println(ip.getSubInfo());
}catch (Exception e){
System.out.println(e);
}
}
}
clean/src/main/resources/api.json 0 → 100644
View file @ 265b180a
{"activity_bargain":{"action":"activity_bargain","detail":{"activity_bargain":{"action_mark":"0","detail_re":"/api/activity/bargain"}},"re":"/api/activity/bargain"},"activity_entry":{"action":"activity_entry","detail":{"activity_entry":{"action_mark":"0","detail_re":"/api/activity/entry"}},"re":"/api/activity/entry"},"activity_get_vip_card":{"action":"activity_get_vip_card","detail":{"activity_get_vip_card":{"action_mark":"0","detail_re":"/api/activity/get_vip_card"}},"re":"/api/activity/get_vip_card"},"activity_intro":{"action":"activity_intro","detail":{"activity_intro":{"action_mark":"0","detail_re":"/api/activity/intro"}},"re":"/api/activity/intro"},"activity_invitation":{"action":"activity_invitation","detail":{"activity_invitation":{"action_mark":"0","detail_re":"/api/activity/invitation"}},"re":"/api/activity/invitation"},"ad_adInfoList":{"action":"ad_adInfoList","detail":{"ad_adInfoList":{"action_mark":"0","detail_re":"/api/ad/adInfoList"}},"re":"/api/ad/adInfoList"},"book_flowInfo":{"action":"book_flowInfo","detail":{"book_flowInfo":{"action_mark":"0","detail_re":"/api/book/flowInfo"}},"re":"/api/book/flowInfo"},"bookcover_bookcoverInfo":{"action":"bookcover_bookcoverInfo","detail":{"bookcover_bookcoverInfo":{"action_mark":"1","detail_re":"/api/bookcover/bookcoverInfo"}},"re":"/api/bookcover/bookcoverInfo"},"bookcover_comment":{"action":"bookcover_comment","detail":{"bookcover_comment":{"action_mark":"1","detail_re":"/api/bookcover/comment"}},"re":"/api/bookcover/comment"},"bookcover_commentList":{"action":"bookcover_commentList","detail":{"bookcover_commentList":{"action_mark":"1","detail_re":"/api/bookcover/commentList"}},"re":"/api/bookcover/commentList"},"bookcover_recommend":{"action":"bookcover_recommend","detail":{"bookcover_recommend":{"action_mark":"1","detail_re":"/api/bookcover/recommend"}},"re":"/api/bookcover/recommend"},"bookcover_reply":{"action":"bookcover_reply","detail":{"bookcover_reply":{"action_mark":"0","detail_re":"/api/bookcover/reply"}},"re":"/api/bookcover/reply"},"bookcover_replyList":{"action":"bookcover_replyList","detail":{"bookcover_replyList":{"action_mark":"0","detail_re":"/api/bookcover/replyList"}},"re":"/api/bookcover/replyList"},"bookcover_shareInfo":{"action":"bookcover_shareInfo","detail":{"bookcover_shareInfo":{"action_mark":"1","detail_re":"/api/bookcover/shareInfo"}},"re":"/api/bookcover/shareInfo"},"bookshelf_addBookshelf":{"action":"bookshelf_addBookshelf","detail":{"bookshelf_addBookshelf":{"action_mark":"1","detail_re":"/api/bookshelf/addBookshelf"}},"re":"/api/bookshelf/addBookshelf"},"bookshelf_bookUpdateInfo":{"action":"bookshelf_bookUpdateInfo","detail":{"bookshelf_bookUpdateInfo":{"action_mark":"2","detail_re":"/api/bookshelf/bookUpdateInfo"}},"re":"/api/bookshelf/bookUpdateInfo"},"bookshelf_delete":{"action":"bookshelf_delete","detail":{"bookshelf_delete":{"action_mark":"0","detail_re":"/api/bookshelf/delete"}},"re":"/api/bookshelf/delete"},"bookshelf_deleteHistory":{"action":"bookshelf_deleteHistory","detail":{"bookshelf_deleteHistory":{"action_mark":"0","detail_re":"/api/bookshelf/deleteHistory"}},"re":"/api/bookshelf/deleteHistory"},"bookshelf_history":{"action":"bookshelf_history","detail":{"bookshelf_history":{"action_mark":"0","detail_re":"/api/bookshelf/history"}},"re":"/api/bookshelf/history"},"bookshelf_noticeList":{"action":"bookshelf_noticeList","detail":{"bookshelf_noticeList":{"action_mark":"0","detail_re":"/api/bookshelf/noticeList"}},"re":"/api/bookshelf/noticeList"},"bookshelf_recommend":{"action":"bookshelf_recommend","detail":{"bookshelf_recommend":{"action_mark":"0","detail_re":"/api/bookshelf/recommend"}},"re":"/api/bookshelf/recommend"},"bookshelf_recommendBook":{"action":"bookshelf_recommendBook","detail":{"bookshelf_recommendBook":{"action_mark":"0","detail_re":"/api/bookshelf/recommendBook"}},"re":"/api/bookshelf/recommendBook"},"bookshelf_recommendBookList":{"action":"bookshelf_recommendBookList","detail":{"bookshelf_recommendBookList":{"action_mark":"0","detail_re":"/api/bookshelf/recommendBookList"}},"re":"/api/bookshelf/recommendBookList"},"bookshelf_update":{"action":"bookshelf_update","detail":{"bookshelf_update":{"action_mark":"1","detail_re":"/api/bookshelf/update"}},"re":"/api/bookshelf/update"},"bookstore_activity_list":{"action":"bookstore_activity_list","detail":{"bookstore_activity_list":{"action_mark":"0","detail_re":"/api/bookstore/activity_list"}},"re":"/api/bookstore/activity_list"},"bookstore_associativeSearch":{"action":"bookstore_associativeSearch","detail":{"bookstore_associativeSearch":{"action_mark":"0","detail_re":"/api/bookstore/associativeSearch"}},"re":"/api/bookstore/associativeSearch"},"bookstore_bookInfoList":{"action":"bookstore_bookInfoList","detail":{"bookstore_bookInfoList":{"action_mark":"0","detail_re":"/api/bookstore/bookInfoList"}},"re":"/api/bookstore/bookInfoList"},"bookstore_bookListChange":{"action":"bookstore_bookListChange","detail":{"bookstore_bookListChange":{"action_mark":"0","detail_re":"/api/bookstore/bookListChange"}},"re":"/api/bookstore/bookListChange"},"bookstore_channelInfo":{"action":"bookstore_channelInfo","detail":{"bookstore_channelInfo":{"action_mark":"0","detail_re":"/api/bookstore/channelInfo"}},"re":"/api/bookstore/channelInfo"},"bookstore_channelList":{"action":"bookstore_channelList","detail":{"bookstore_channelList":{"action_mark":"0","detail_re":"/api/bookstore/channelList"}},"re":"/api/bookstore/channelList"},"bookstore_classifyList":{"action":"bookstore_classifyList","detail":{"bookstore_classifyList":{"action_mark":"0","detail_re":"/api/bookstore/classifyList"}},"re":"/api/bookstore/classifyList"},"bookstore_editorList":{"action":"bookstore_editorList","detail":{"bookstore_editorList":{"action_mark":"0","detail_re":"/api/bookstore/editorList"}},"re":"/api/bookstore/editorList"},"bookstore_editorRecommendList":{"action":"bookstore_editorRecommendList","detail":{"bookstore_editorRecommendList":{"action_mark":"0","detail_re":"/api/bookstore/editorRecommendList"}},"re":"/api/bookstore/editorRecommendList"},"bookstore_hotSearch":{"action":"bookstore_hotSearch","detail":{"bookstore_hotSearch":{"action_mark":"0","detail_re":"/api/bookstore/hotSearch"}},"re":"/api/bookstore/hotSearch"},"bookstore_ranking":{"action":"bookstore_ranking","detail":{"bookstore_ranking":{"action_mark":"0","detail_re":"/api/bookstore/ranking"}},"re":"/api/bookstore/ranking"},"bookstore_search":{"action":"bookstore_search","detail":{"bookstore_search":{"action_mark":"0","detail_re":"/api/bookstore/search"}},"re":"/api/bookstore/search"},"operation_eventDialogList":{"action":"operation_eventDialogList","detail":{"operation_eventDialogList":{"action_mark":"0","detail_re":"/api/operation/eventDialogList"}},"re":"/api/operation/eventDialogList"},"operation_getBooksForNewUser":{"action":"operation_getBooksForNewUser","detail":{"operation_getBooksForNewUser":{"action_mark":"0","detail_re":"/api/operation/getBooksForNewUser"}},"re":"/api/operation/getBooksForNewUser"},"operation_getComerGiftInfo":{"action":"operation_getComerGiftInfo","detail":{"operation_getComerGiftInfo":{"action_mark":"0","detail_re":"/api/operation/getComerGiftInfo"}},"re":"/api/operation/getComerGiftInfo"},"operation_getOperatingActivitiesList":{"action":"operation_getOperatingActivitiesList","detail":{"operation_getOperatingActivitiesList":{"action_mark":"0","detail_re":"/api/operation/getOperatingActivitiesList"}},"re":"/api/operation/getOperatingActivitiesList"},"operation_getScrollsForCommand":{"action":"operation_getScrollsForCommand","detail":{"operation_getScrollsForCommand":{"action_mark":"0","detail_re":"/api/operation/getScrollsForCommand"}},"re":"/api/operation/getScrollsForCommand"},"operation_incentiveVideoGift":{"action":"operation_incentiveVideoGift","detail":{"operation_incentiveVideoGift":{"action_mark":"0","detail_re":"/api/operation/incentiveVideoGift"}},"re":"/api/operation/incentiveVideoGift"},"operation_interactAd":{"action":"operation_interactAd","detail":{"operation_interactAd":{"action_mark":"0","detail_re":"/api/operation/interactAd"}},"re":"/api/operation/interactAd"},"operation_privilegesForNewUser":{"action":"operation_privilegesForNewUser","detail":{"operation_privilegesForNewUser":{"action_mark":"0","detail_re":"/api/operation/privilegesForNewUser"}},"re":"/api/operation/privilegesForNewUser"},"operation_statistics":{"action":"operation_statistics","detail":{"operation_statistics":{"action_mark":"0","detail_re":"/api/operation/statistics"}},"re":"/api/operation/statistics"},"other_actionLog":{"action":"other_actionLog","detail":{"other_actionLog":{"action_mark":"0","detail_re":"/api/other/actionLog"}},"re":"/api/other/actionLog"},"other_config":{"action":"other_config","detail":{"other_config":{"action_mark":"0","detail_re":"/api/other/config"}},"re":"/api/other/config"},"other_newUserActionLog":{"action":"other_newUserActionLog","detail":{"other_newUserActionLog":{"action_mark":"0","detail_re":"/api/other/newUserActionLog"}},"re":"/api/other/newUserActionLog"},"other_phoneInfo":{"action":"other_phoneInfo","detail":{"other_phoneInfo":{"action_mark":"0","detail_re":"/api/other/phoneInfo"}},"re":"/api/other/phoneInfo"},"other_portrayal":{"action":"other_portrayal","detail":{"other_portrayal":{"action_mark":"0","detail_re":"/api/other/portrayal"}},"re":"/api/other/portrayal"},"other_readFeedback":{"action":"other_readFeedback","detail":{"other_readFeedback":{"action_mark":"2","detail_re":"/api/other/readFeedback"}},"re":"/api/other/readFeedback"},"read_adChapter":{"action":"read_adChapter","detail":{"read_adChapter":{"action_mark":"0","detail_re":"/api/read/adChapter"}},"re":"/api/read/adChapter"},"read_addHistory":{"action":"read_addHistory","detail":{"read_addHistory":{"action_mark":"2","detail_re":"/api/read/addHistory"}},"re":"/api/read/addHistory"},"read_buyBatchChapter":{"action":"read_buyBatchChapter","detail":{"read_buyBatchChapter":{"action_mark":"2","detail_re":"/api/read/buyBatchChapter"}},"re":"/api/read/buyBatchChapter"},"read_buyBook":{"action":"read_buyBook","detail":{"read_buyBook":{"action_mark":"1","detail_re":"/api/read/buyBook"}},"re":"/api/read/buyBook"},"read_buyChapter":{"action":"read_buyChapter","detail":{"read_buyChapter":{"action_mark":"2","detail_re":"/api/read/buyChapter"}},"re":"/api/read/buyChapter"},"read_cacheChapterInfo":{"action":"read","detail":{"read_cacheChapterInfo":{"action_mark":"2","detail_re":"/api/read/cacheChapterInfo"}},"re":"/api/read/cacheChapterInfo"},"read_chapterInfo":{"action":"read","detail":{"read_chapterInfo":{"action_mark":"2","detail_re":"/api/read/chapterInfo"}},"re":"/api/read/chapterInfo"},"read_directory":{"action":"read_directory","detail":{"read_directory":{"action_mark":"0","detail_re":"/api/read/directory"}},"re":"/api/read/directory"},"read_endChangeMoreInfo":{"action":"read_endChangeMoreInfo","detail":{"read_endChangeMoreInfo":{"action_mark":"1","detail_re":"/api/read/endChangeMoreInfo"}},"re":"/api/read/endChangeMoreInfo"},"read_endInfo":{"action":"read_endInfo","detail":{"read_endInfo":{"action_mark":"1","detail_re":"/api/read/endInfo"}},"re":"/api/read/endInfo"},"read_getBuyChapterInfo":{"action":"read_getBuyChapterInfo","detail":{"read_getBuyChapterInfo":{"action_mark":"1","detail_re":"/api/read/getBuyChapterInfo"}},"re":"/api/read/getBuyChapterInfo"},"read_getChapterPayInfo":{"action":"read_getChapterPayInfo","detail":{"read_getChapterPayInfo":{"action_mark":"0","detail_re":"/api/read/getChapterPayInfo"}},"re":"/api/read/getChapterPayInfo"},"read_getHistory":{"action":"read_getHistory","detail":{"read_getHistory":{"action_mark":"1","detail_re":"/api/read/getHistory"}},"re":"/api/read/getHistory"},"read_jumpChapterInfo":{"action":"read_jumpChapterInfo","detail":{"read_jumpChapterInfo":{"action_mark":"1","detail_re":"/api/read/jumpChapterInfo"}},"re":"/api/read/jumpChapterInfo"},"read_like":{"action":"read_like","detail":{"read_like":{"action_mark":"0","detail_re":"/api/read/like"}},"re":"/api/read/like"},"read_reward":{"action":"read_reward","detail":{"read_reward":{"action_mark":"1","detail_re":"/api/read/reward"}},"re":"/api/read/reward"},"read_rewardInfo":{"action":"read_rewardInfo","detail":{"read_rewardInfo":{"action_mark":"1","detail_re":"/api/read/rewardInfo"}},"re":"/api/read/rewardInfo"},"read_urge":{"action":"read_urge","detail":{"read_urge":{"action_mark":"2","detail_re":"/api/read/urge"}},"re":"/api/read/urge"},"recharge_aliPayInfo":{"action":"recharge_aliPayInfo","detail":{"recharge_aliPayInfo":{"action_mark":"0","detail_re":"/api/recharge/aliPayInfo"}},"re":"/api/recharge/aliPayInfo"},"recharge_aliPayInfoMonthly":{"action":"recharge_aliPayInfoMonthly","detail":{"recharge_aliPayInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/aliPayInfoMonthly"}},"re":"/api/recharge/aliPayInfoMonthly"},"recharge_cancelAliPayInfo":{"action":"recharge_cancelAliPayInfo","detail":{"recharge_cancelAliPayInfo":{"action_mark":"0","detail_re":"/api/recharge/cancelAliPayInfo"}},"re":"/api/recharge/cancelAliPayInfo"},"recharge_cancelWeChatPayInfo":{"action":"recharge_cancelWeChatPayInfo","detail":{"recharge_cancelWeChatPayInfo":{"action_mark":"0","detail_re":"/api/recharge/cancelWeChatPayInfo"}},"re":"/api/recharge/cancelWeChatPayInfo"},"recharge_changeInfo":{"action":"recharge_changeInfo","detail":{"recharge_changeInfo":{"action_mark":"0","detail_re":"/api/recharge/changeInfo"}},"re":"/api/recharge/changeInfo"},"recharge_changeInfoMonthly":{"action":"recharge_changeInfoMonthly","detail":{"recharge_changeInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/changeInfoMonthly"}},"re":"/api/recharge/changeInfoMonthly"},"recharge_iOS_changeInfoMonthly":{"action":"recharge_iOS_changeInfoMonthly","detail":{"recharge_iOS_changeInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/iOS/changeInfoMonthly"}},"re":"/api/recharge/iOS/changeInfoMonthly"},"recharge_iOS_iapVerifyMonthly":{"action":"recharge_iOS_iapVerifyMonthly","detail":{"recharge_iOS_iapVerifyMonthly":{"action_mark":"0","detail_re":"/api/recharge/iOS/iapVerifyMonthly"}},"re":"/api/recharge/iOS/iapVerifyMonthly"},"recharge_iOS_returnDiscount":{"action":"recharge_iOS_returnDiscount","detail":{"recharge_iOS_returnDiscount":{"action_mark":"0","detail_re":"/api/recharge/iOS/returnDiscount"}},"re":"/api/recharge/iOS/returnDiscount"},"recharge_iOSchangeInfo":{"action":"recharge_iOSchangeInfo","detail":{"recharge_iOSchangeInfo":{"action_mark":"0","detail_re":"/api/recharge/iOS/changeInfo"}},"re":"/api/recharge/iOS/changeInfo"},"recharge_iOSiapVerify":{"action":"recharge_iOSiapVerify","detail":{"recharge_iOSiapVerify":{"action_mark":"0","detail_re":"/api/recharge/iOS/iapVerify"}},"re":"/api/recharge/iOS/iapVerify"},"recharge_paynowInfo":{"action":"recharge_paynowInfo","detail":{"recharge_paynowInfo":{"action_mark":"0","detail_re":"/api/recharge/paynowInfo"}},"re":"/api/recharge/paynowInfo"},"recharge_pub_wechat_pay_info":{"action":"recharge_pub_wechat_pay_info","detail":{"recharge_pub_wechat_pay_info":{"action_mark":"0","detail_re":"/api/recharge/pub_wechat_pay_info"}},"re":"/api/recharge/pub_wechat_pay_info"},"recharge_returnDiscount":{"action":"recharge_returnDiscount","detail":{"recharge_returnDiscount":{"action_mark":"0","detail_re":"/api/recharge/returnDiscount"}},"re":"/api/recharge/returnDiscount"},"recharge_weChatPayInfo":{"action":"recharge_weChatPayInfo","detail":{"recharge_weChatPayInfo":{"action_mark":"0","detail_re":"/api/recharge/weChatPayInfo"}},"re":"/api/recharge/weChatPayInfo"},"recharge_weChatPayInfoMonthly":{"action":"recharge_weChatPayInfoMonthly","detail":{"recharge_weChatPayInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/weChatPayInfoMonthly"}},"re":"/api/recharge/weChatPayInfoMonthly"},"recharge_wechat_h5_pay_info":{"action":"recharge_wechat_h5_pay_info","detail":{"recharge_wechat_h5_pay_info":{"action_mark":"0","detail_re":"/api/recharge/wechat_h5_pay_info"}},"re":"/api/recharge/wechat_h5_pay_info"},"recharge_wechat_h5_pay_result_info":{"action":"recharge_wechat_h5_pay_result_info","detail":{"recharge_wechat_h5_pay_result_info":{"action_mark":"0","detail_re":"/api/recharge/wechat_h5_pay_result_info"}},"re":"/api/recharge/wechat_h5_pay_result_info"},"task_common_config":{"action":"task_common_config","detail":{"task_common_config":{"action_mark":"0","detail_re":"/api/task/v1/common/config"}},"re":"/api/task/v1/common/config\r\n"},"task_common_notice":{"action":"task_common_notice","detail":{"task_common_notice":{"action_mark":"0","detail_re":"/api/task/v1/common/notice"}},"re":"/api/task/v1/common/notice"},"task_common_qiniu_token":{"action":"task_common_qiniu_token","detail":{"task_common_qiniu_token":{"action_mark":"0","detail_re":"/api/task/v1/common/qiniu/token"}},"re":"/api/task/v1/common/qiniu/token"},"task_common_text_config":{"action":"task_common_text_config","detail":{"task_common_text_config":{"action_mark":"0","detail_re":"/api/task/v1/common/text/config"}},"re":"/api/task/v1/common/text/config"},"task_complete":{"action":"task_complete","detail":{"task_complete":{"action_mark":"0","detail_re":"/api/task/complete"}},"re":"/api/task/complete"},"task_getList":{"action":"task_getList","detail":{"task_getList":{"action_mark":"0","detail_re":"/api/task/getList"}},"re":"/api/task/getList"},"task_getTaskInfo":{"action":"task_getTaskInfo","detail":{"task_getTaskInfo":{"action_mark":"0","detail_re":"/api/task/getTaskInfo"}},"re":"/api/task/getTaskInfo"},"task_share_user_bind":{"action":"task_share_user_bind","detail":{"task_share_user_bind":{"action_mark":"0","detail_re":"/api/task/v1/share/user/bind"}},"re":"/api/task/v1/share/user/bind"},"task_share_user_share":{"action":"task_share_user_share","detail":{"task_share_user_share":{"action_mark":"0","detail_re":"/api/task/v1/share/user/share"}},"re":"/api/task/v1/share/user/share"},"task_task_finish":{"action":"task_task_finish","detail":{"task_task_finish":{"action_mark":"0","detail_re":"/api/task/v1/task/finish"}},"re":"/api/task/v1/task/finish"},"task_task_list":{"action":"task_task_list","detail":{"task_task_list":{"action_mark":"0","detail_re":"/api/task/v1/task/list"}},"re":"/api/task/v1/task/list"},"task_wallet_user_info":{"action":"task_wallet_user_info","detail":{"task_wallet_user_info":{"action_mark":"0","detail_re":"/api/task/v1/wallet/user/info"}},"re":"/api/task/v1/wallet/user/info"},"task_wallet_user_records":{"action":"task_wallet_user_records","detail":{"task_wallet_user_records":{"action_mark":"0","detail_re":"/api/task/v1/wallet/user/records"}},"re":"/api/task/v1/wallet/user/records"},"task_wallet_user_share_records":{"action":"task_wallet_user_share_records","detail":{"task_wallet_user_share_records":{"action_mark":"0","detail_re":"/api/task/v1/wallet/user/share/records"}},"re":"/api/task/v1/wallet/user/share/records"},"task_withdraw_list":{"action":"task_withdraw_list","detail":{"task_withdraw_list":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/list"}},"re":"/api/task/v1/withdraw/list"},"task_withdraw_records":{"action":"task_withdraw_records","detail":{"task_withdraw_records":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/records"}},"re":"/api/task/v1/withdraw/records"},"task_withdraw_submit":{"action":"task_withdraw_submit","detail":{"task_withdraw_submit":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/submit"}},"re":"/api/task/v1/withdraw/submit"},"task_withdraw_valid":{"action":"task_withdraw_valid","detail":{"task_withdraw_valid":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/valid"}},"re":"/api/task/v1/withdraw/valid"},"user_Register":{"action":"user_Register","detail":{"user_Register":{"action_mark":"0","detail_re":"/api/user/autoRegister"}},"re":"/api/user/autoRegister"},"user_bindPhoneGiftBagInfo":{"action":"user_bindPhoneGiftBagInfo","detail":{"user_bindPhoneGiftBagInfo":{"action_mark":"0","detail_re":"/api/user/bindPhoneGiftBagInfo"}},"re":"/api/user/bindPhoneGiftBagInfo"},"user_changePhoneNum":{"action":"user_changePhoneNum","detail":{"user_changePhoneNum":{"action_mark":"0","detail_re":"/api/user/changePhoneNum"}},"re":"/api/user/changePhoneNum"},"user_coinList":{"action":"user_coinList","detail":{"user_coinList":{"action_mark":"0","detail_re":"/api/user/coinList"}},"re":"/api/user/coinList"},"user_consumeInfo":{"action":"user_consumeInfo","detail":{"user_consumeInfo":{"action_mark":"0","detail_re":"/api/user/consumeInfo"}},"re":"/api/user/consumeInfo"},"user_feedback":{"action":"user_feedback","detail":{"user_feedback":{"action_mark":"0","detail_re":"/api/feedback"}},"re":"/api/feedback"},"user_getPublicKey":{"action":"user_getPublicKey","detail":{"user_getPublicKey":{"action_mark":"0","detail_re":"/api/getPublicKey"}},"re":"/api/getPublicKey"},"user_h5_thirdLogin":{"action":"user_h5_thirdLogin","detail":{"user_h5_thirdLogin":{"action_mark":"0","detail_re":"/api/user/h5/thirdLogin"}},"re":"/api/user/h5/thirdLogin"},"user_modifyUserInfo":{"action":"user_modifyUserInfo","detail":{"user_modifyUserInfo":{"action_mark":"0","detail_re":"/api/user/modifyUserInfo"}},"re":"/api/user/modifyUserInfo"},"user_msgAuthCode":{"action":"user_msgAuthCode","detail":{"user_msgAuthCode":{"action_mark":"0","detail_re":"/api/user/msgAuthCode"}},"re":"/api/user/msgAuthCode"},"user_openAd":{"action":"user_openAd","detail":{"user_openAd":{"action_mark":"0","detail_re":"/api/openAd"}},"re":"/api/openAd"},"user_phoneNumLogin":{"action":"user_phoneNumLogin","detail":{"user_phoneNumLogin":{"action_mark":"0","detail_re":"/api/user/phoneNumLogin"}},"re":"/api/user/phoneNumLogin"},"user_preference":{"action":"user_preference","detail":{"user_preference":{"action_mark":"0","detail_re":"/api/user/preference"}},"re":"/api/user/preference"},"user_question":{"action":"user_question","detail":{"user_question":{"action_mark":"0","detail_re":"/api/user/question"}},"re":"/api/user/question"},"user_rechargeInfo":{"action":"user_rechargeInfo","detail":{"user_rechargeInfo":{"action_mark":"0","detail_re":"/api/user/rechargeInfo"}},"re":"/api/user/rechargeInfo"},"user_setPreference":{"action":"user_setPreference","detail":{"user_setPreference":{"action_mark":"0","detail_re":"/api/user/setPreference"}},"re":"/api/user/setPreference"},"user_shareResult":{"action":"user_shareResult","detail":{"user_shareResult":{"action_mark":"0","detail_re":"/api/user/shareResult"}},"re":"/api/user/shareResult"},"user_signGift":{"action":"user_signGift","detail":{"user_signGift":{"action_mark":"0","detail_re":"/api/user/signGift"}},"re":"/api/user/signGift"},"user_signIn":{"action":"user_signIn","detail":{"user_signIn":{"action_mark":"0","detail_re":"/api/user/signIn"}},"re":"/api/user/signIn"},"user_signInInfo":{"action":"user_signInInfo","detail":{"user_signInInfo":{"action_mark":"0","detail_re":"/api/user/signInInfo"}},"re":"/api/user/signInInfo"},"user_thirdLogin":{"action":"user_thirdLogin","detail":{"user_thirdLogin":{"action_mark":"0","detail_re":"/api/user/thirdLogin"}},"re":"/api/user/thirdLogin"},"user_unbindThirdLogin":{"action":"user_unbindThirdLogin","detail":{"user_unbindThirdLogin":{"action_mark":"0","detail_re":"/api/user/unbindThirdLogin"}},"re":"/api/user/unbindThirdLogin"},"user_update":{"action":"user_update","detail":{"user_update":{"action_mark":"0","detail_re":"/api/update"}},"re":"/api/update"},"user_userInfo":{"action":"user_userInfo","detail":{"user_userInfo":{"action_mark":"0","detail_re":"/api/user/userInfo"}},"re":"/api/user/userInfo"}}
clean/src/main/resources/clean_api.py 0 → 100644
View file @ 265b180a
#!/usr/bin/env python
# -* - coding: UTF-8 -* -
#name : by demon
#time : 2017-05-24
import sys
import re
import time
import os
import datetime
import socket,json
import urllib
import traceback
from struct import pack, unpack
reload(sys)
sys.setdefaultencoding('utf-8')
class clearurl():
#@staticmethod
def __init__(self, re_path):
self.re_path = re_path
self.re_mess = os.popen('hadoop fs -cat %s' %re_path).read()
self.dict_re=eval(self.re_mess)
def clear_api(self,requrlstr,reqParam):
dict_re = self.dict_re
#定义变量
action="-999"
action_detail="-999"
bookid="-999"
chapterNum="-999"
querymessage="-999"
requrl = requrlstr.split("?")[0]
for i in dict_re.keys():
if 're' not in dict_re[i]:
continue
patterni = re.compile(r'^%s' % dict_re[i]['re'])
break_flag = False
if patterni.match(requrl):
for n in dict_re[i]['detail'].keys():
patternn = re.compile(r'^%s' % dict_re[i]['detail'][n]['detail_re'])
if patternn.match(requrl):
break_flag = True
action = dict_re[i]['action']
action_detail = n
action_mark = dict_re[i]['detail'][n]['action_mark']
try:
if action == 'search':
querystr = requrlstr.split('query=')[1].split('&')[0]
querymessage = urllib.unquote(querystr)
else:
querymessage = '-999'
except:
querymessage='-999'
#print action_mark,reqParam
try:
if action_mark == "0":
bookid = "-999"
chapterNum = "-999"
elif action_mark == "1":
reqjson=json.loads(reqParam)
if 'bookIds' in reqParam:
bookid=reqjson['bookIds']
elif 'bookId' in reqParam:
bookid=reqjson['bookId']
else:
bookid='-999'
chapterNum="-999"
elif action_mark == "2":
if reqParam !='':
reqjson = json.loads(reqParam)
if 'bookIds' in reqParam:
bookid=reqjson['bookIds']
elif 'bookId' in reqParam:
bookid=reqjson['bookId']
else:
bookid='-999'
if 'chapterNum' in reqParam:
chapterNum = reqjson['chapterNum']
else:
chapterNum="-999"
else:
bookid = "-999"
chapterNum = "-999"
except:
bookid = "-999"
chapterNum = "-999"
break
if break_flag:
break
return action, action_detail, bookid, chapterNum,querymessage
def clear_refer(self,refer,reqParam,platform):
# print refer
try:
refer_com = refer.split("/")[2]
pattern_com = re.compile(r'[a-z]+.yc.ifeng.com')
if pattern_com.match(refer_com):
refer_url = refer.split(".com")[1].split("?")[0]
#print refer_url
try:
if platform == "api":
referaction, referaction_detail, referbookid, referchapterNum,querymessage = self.clear_api(refer_url,reqParam)
except:
referaction = "-999"
else:
referaction = "-999"
except:
refer_com = "-999"
referaction = "-999"
return refer_com, referaction
class IPInfo():
'''QQWry.Dat数据库查询功能集合
'''
def __init__(self, dbname):
''' 初始化类，读取数据库内容为一个字符串，
通过开始8字节确定数据库的索引信息'''
self.dbname = dbname
# f = file(dbname, 'r')
# Demon注：在Windows下用'r'会有问题，会把\r\n转换成\n
# 详见http://demon.tw/programming/python-open-mode.html
# 还有Python文档中不提倡用file函数来打开文件，推荐用open
f = open(dbname, 'rb')
self.img = f.read()
f.close()
# QQWry.Dat文件的开始8字节是索引信息,前4字节是开始索引的偏移值，
# 后4字节是结束索引的偏移值。
# (self.firstIndex, self.lastIndex) = unpack('II', self.img[:8])
# Demon注：unpack默认使用的endian是和机器有关的
# Intel x86和AMD64(x86-64)是little-endian
# Motorola 68000和PowerPC G5是big-endian
# 而纯真数据库全部采用了little-endian字节序
# 所以在某些big-endian的机器上原代码会出错
(self.firstIndex, self.lastIndex) = unpack('<II', self.img[:8])
# 每条索引长7字节，这里得到索引总个数
self.indexCount = (self.lastIndex - self.firstIndex) / 7 + 1
def getString(self, offset=0):
''' 读取字符串信息，包括"国家"信息和"地区"信息
QQWry.Dat的记录区每条信息都是一个以'\0'结尾的字符串'''
o2 = self.img.find('\0', offset)
# return self.img[offset:o2]
# 有可能只有国家信息没有地区信息，
gb2312_str = self.img[offset:o2]
try:
utf8_str = unicode(gb2312_str, 'gb2312').encode('utf-8')
except:
return '未知'
return utf8_str
def getLong3(self, offset=0):
'''QQWry.Dat中的偏移记录都是3字节，本函数取得3字节的偏移量的常规表示
QQWry.Dat使用"字符串"存储这些值'''
s = self.img[offset: offset + 3]
s += '\0'
# unpack用一个'I'作为format，后面的字符串必须是4字节
# return unpack('I', s)[0]
# Demon注：和上面一样，强制使用little-endian
return unpack('<I', s)[0]
def getAreaAddr(self, offset=0):
''' 通过给出偏移值，取得区域信息字符串，'''
byte = ord(self.img[offset])
if byte == 1 or byte == 2:
# 第一个字节为1或者2时，取得2-4字节作为一个偏移量调用自己
p = self.getLong3(offset + 1)
return self.getAreaAddr(p)
else:
return self.getString(offset)
def getAddr(self, offset, ip=0):
img = self.img
o = offset
byte = ord(img[o])
if byte == 1:
# 重定向模式1
# [IP][0x01][国家和地区信息的绝对偏移地址]
# 使用接下来的3字节作为偏移量调用字节取得信息
return self.getAddr(self.getLong3(o + 1))
if byte == 2:
# 重定向模式2
# [IP][0x02][国家信息的绝对偏移][地区信息字符串]
# 使用国家信息偏移量调用自己取得字符串信息
cArea = self.getAreaAddr(self.getLong3(o + 1))
o += 4
# 跳过前4字节取字符串作为地区信息
aArea = self.getAreaAddr(o)
return (cArea, aArea)
if byte != 1 and byte != 2:
# 最简单的IP记录形式，[IP][国家信息][地区信息]
# 重定向模式1有种情况就是偏移量指向包含国家和地区信息两个字符串
# 即偏移量指向的第一个字节不是1或2,就使用这里的分支
# 简单地说：取连续取两个字符串！
cArea = self.getString(o)
# o += len(cArea) + 1
# 我们已经修改cArea为utf-8字符编码了，len取得的长度会有变，
# 用下面方法得到offset
o = self.img.find('\0', o) + 1
aArea = self.getString(o)
return (cArea, aArea)
def find(self, ip, l, r):
''' 使用二分法查找网络字节编码的IP地址的索引记录'''
if r - l <= 1:
return l
m = (l + r) / 2
o = self.firstIndex + m * 7
# new_ip = unpack('I', self.img[o: o+4])[0]
# Demon注：和上面一样，强制使用little-endian
new_ip = unpack('<I', self.img[o: o + 4])[0]
if ip <= new_ip:
return self.find(ip, l, m)
else:
return self.find(ip, m, r)
def getIPAddr(self, ip):
''' 调用其他函数，取得信息！'''
# 使用网络字节编码IP地址
ip = unpack('!I', socket.inet_aton(ip))[0]
# 使用 self.find 函数查找ip的索引偏移
i = self.find(ip, 0, self.indexCount - 1)
# 得到索引记录
o = self.firstIndex + i * 7
# 索引记录格式是： 前4字节IP信息+3字节指向IP记录信息的偏移量
# 这里就是使用后3字节作为偏移量得到其常规表示（QQWry.Dat用字符串表示值）
o2 = self.getLong3(o + 4)
# IP记录偏移值+4可以丢弃前4字节的IP地址信息。
(c, a) = self.getAddr(o2 + 4)
return (c, a)
def output(self, first, last):
for i in range(first, last):
o = self.firstIndex + i * 7
ip = socket.inet_ntoa(pack('!I', unpack('I', self.img[o:o + 4])[0]))
offset = self.getLong3(o + 4)
(c, a) = self.getAddr(offset + 4)
print "%s %d %s/%s" % (ip, offset, c, a)
# def getIP(self,address):
# i = IPInfo(self.dbname)
# (c, a) = i.getIPAddr(address)
# Demon注：如果是在Windows命令行中运行把编码转回gb2312以避免乱码
# if sys.platform == 'win32':
# c = unicode(c, 'utf-8').encode('gb2312')
# a = unicode(a, 'utf-8').encode('gb2312')
return c
def read_input(file):
for line in file:
yield line.strip()
def main():
platform = "api"
#加载清洗url脚本
clear = clearurl('/newread/tool/clean/dictre/%s.re' %platform)
#加载IPSeek
if os.path.isfile('/tmp/QQWry.Dat'):
pass
else:
os.system('hadoop fs -get /newread/tool/clean/QQWry.Dat /tmp/')
#加载IPSeek
P = IPInfo('/tmp/QQWry.Dat')
#清洗程序开始
data = read_input(sys.stdin)
for line in data:
try:
list = line.split("] [")
#切割日志各字段
logversion = list[0].split("[")[1]
projectname = list[1]
serverip = list[2]
reqbegintime = list[3]
reqendtime = list[4]
reqspendtime = list[5]
reqip = list[6]
requrlstr = list[7]
#过滤下载日志 add by zhaopf 2018-12-06
if(('/api/download/' in requrlstr) or ('/api/other/jrttBack' in requrlstr) or ('/kuaishou/notify' in requrlstr) or ('/api/getPublicKey' in requrlstr)):
continue
refer = list[8]
method = list[9]
httpcode = list[10]
userid = list[11]
sessionid = list[12]
platformid = list[13]
if( 'type=bbs' in requrlstr):
if platformid == '1':
platformid=100
elif platformid == '2':
platformid=200
appversion = list[14]
channelid = list[15]
bookrecommend= list[16]
payType = list[17]
paynum = list[18]
rechargetype = list[19]
payamount = list[20]
ssid = list[21]
isnewuser = list[22]
ischannelnewuser = list[23]
readwordstype = list[24]
readwords = list[25]
mobile = list[26]
uaplatform = list[27]
uastat = list[28]
apicode = list[29]
xClient = list[30]
ua = list[31]
if('JianKongBao' in ua ):
continue
#print(list[33])
reqParam = list[32]
processParam = list[33]
#clear refer 定义来源网址以及referaction_detail
refer_com, referaction_detail = clear.clear_refer(refer,reqParam,platform)
#将用户IP转化为省份以及地区
try:
UserIp = reqip.split(',')[0]
(country, detail_area) = P.getIPAddr(UserIp)
except:
country="-999"
detail_area="-999"
#调用clear方法，将URL转化为action等
#定义变量
action="-999"
action_detail="-999"
bookid="-999"
chapterNum="-999"
querymessage="-999"
#try:
if platform =="api":
action, action_detail, bookid, chapterNum ,querymessage = clear.clear_api(requrlstr, reqParam)
#except:
# action = "-999"
# action_detail = "-999"
# bookid = "-999"
# chapterNum = "-999"
# querymessage = '-999'
#切分Xclient取出相应的数据
try:
sdk = xClient.split('sv=')[1].split(';')[0]
except:
sdk = "-999"
try:
screensize = xClient.split('ss=')[1].split(';')[0]
except:
screensize = "-999"
try:
type = xClient.split('pm=')[1].split(';')[0]
type=urllib.unquote(type)
except:
type = "-999"
try:
imsi = xClient.split('imsi=')[1].split(';')[0]
except:
imsi = "-999"
try:
version = xClient.split('version=')[1].split(';')[0]
if version is not None and version.find('.') > 0:
#print(version)
first_index = version.find('.') + 1
#print(first_index)
last_index = version.rfind('.')
#print(last_index)
appversion2 = version[first_index:last_index]
#print(appversion2)
except:
version = "-999"
appversion2 = "-999"
try:
mac = xClient.split('mac=')[1].split(';')[0]
except:
mac = "-999"
try:
device_id = xClient.split('dID=')[1].split(';')[0]
except:
device_id = "-999"
rn = "-999"
tdcn = "-999"
try:
imei = xClient.split('imei=')[1].split(';')[0]
except:
imei = '-999'
try:
idfa = xClient.split('idfa=')[1].split(';')[0]
except:
idfa = '-999'
#定义浏览器
endbrand = "-999"
browserbrand = "-999"
browserversion = "-999"
primaryurlpath = "-999"
others = querymessage
cellid = "-999"
projectType="-999"
print '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t' %(logversion,projectname,serverip,reqbegintime,reqendtime,reqspendtime,reqip,country,detail_area,requrlstr,action,action_detail,bookid,chapterNum,refer,refer_com,referaction_detail,method,httpcode,userid,sessionid,platformid,appversion,channelid,bookrecommend,payType,paynum,rechargetype,payamount,ssid,isnewuser,ischannelnewuser,readwordstype,readwords,mobile,uaplatform,uastat,apicode,imei,sdk,screensize,type,imsi,cellid,version,device_id,mac,endbrand,browserbrand,browserversion,ua,primaryurlpath,reqParam,others,processParam,projectType,idfa,appversion2)
except Exception, e:
exc_type, exc_obj, exc_tb = sys.exc_info();
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
#print("line="+line)
#print(exc_type, fname, exc_tb.tb_lineno)
#traceback.print_exc()
continue
if __name__ == "__main__":
main()
clean/src/main/resources/clean_api.sh 0 → 100644
View file @ 265b180a
#!/bin/sh
#time 2017-06-07
################# 初始化变量#########################
hadoop="/usr/bin/hadoop";
hdfs="/usr/bin/hdfs";
cleanpath="/home/hdfs/hadoop-hdfs/personas/clean";
if [ "$1" != "" ];then
date=`date -d "$1" +%Y-%m-%d`;
logdate=`date -d "$1" +%Y%m%d`;
else
date=`date -d "-1 day" +%Y-%m-%d`;
logdate=`date -d "-1 day" +%Y%m%d`;
fi
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_android.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_android_lbb.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_android_kyy.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_ios.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_h5.sh $date
clean/src/main/resources/clean_api_android.sh 0 → 100644
View file @ 265b180a
#!/bin/sh
#time 2017-06-07
################# 初始化变量#########################
source /etc/bi.profile
hadoop="/usr/bin/hadoop";
hdfs="/usr/bin/hdfs";
cleanpath="/home/hdfs/hadoop-hdfs/personas/clean";
if [ "$1" != "" ];then
date=`date -d "$1" +%Y-%m-%d`;
logdate=`date -d "$1" +%Y%m%d`;
else
date=`date -d "-1 day" +%Y-%m-%d`;
logdate=`date -d "-1 day" +%Y%m%d`;
fi
year=${date:0:4}
month=${date:4:2}
export MYSQL_BI_IP=10.90.8.32
export MYSQL_BI_USER=readtongji
export MYSQL_BI_PWD=uZpvSuDuEiEOwCtk21on
export MYSQL_BI_DB=db_bi
export MYSQL_BI_PORT=3306
function Delete(){
hadoop fs -rm -r -f /newread/read/log_access_txt/platform=android/day=${date}/
}
function Clean(){
cd ${cleanpath}
mark=`hadoop fs -ls /newread/logflag/android/${logdate}/*.lock | wc -l`
title="android日志检查"
content="${date}android日志上传成功"
content1="${date}android日志上传失败"
flag=0
if [ $mark -eq 1 ];then
/bin/python ${alert_path}/alert_hive.py $title $content "android"
flag=1
else
/bin/python ${alert_path}/alert_hive.py $title $content1 "android"
fi
while [ $mark -eq 0 ];do
sleep 10
mark=`hadoop fs -ls /newread/logflag/android/${logdate}/*.lock | wc -l`
done
echo ${mark}
if [ $mark -eq 1 -a $flag -eq 0 ];then
/bin/python ${alert_path}/alert_hive.py $title $content "android"
elif [ $mark -eq 0 -a $flag -eq 0 ];then
/bin/python ${alert_path}/alert_hive.py $title $content1 "android"
fi
hadoop jar /usr/hdp/2.6.5.0-292/hadoop-mapreduce/hadoop-streaming.jar -input /newread/logbak/android/${logdate}/*-access-*.log -output /newread/read/log_access_txt/platform=android/day=${date} -file /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py -mapper "/home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py" -jobconf mapred.job.name="job_acclog_last_and"
#hadoop jar /usr/hdp/2.6.5.0-292/hadoop-mapreduce/hadoop-streaming.jar -inputformat com.hadoop.mapred.DeprecatedLzoTextInputFormat -input /newread/logbak/android/${logdate}/*-access-*.log.lzo -output /newread/read/log_access_txt/platform=android/day=${date} -file /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py -mapper "/home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py" -jobconf mapred.job.name="job_acclog_last_and" -jobconf mapred.output.compress=true -jobconf mapred.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec
#check_result "clean_acclog_android"
}
function LoadHive(){
hive <<EOF
use read;
ALTER TABLE log_access_txt DROP IF EXISTS partition (platform='android',day='$date');
alter table log_access_txt add IF NOT EXISTS partition (platform='android',day='$date') location '/newread/read/log_access_txt/platform=android/day=$date';
exit;
EOF
exit;
}
function main(){
#hadoop fs -test -e /newread/logbak/novel/$logdate/access/
#while [ $? -ne 0 ];do
# sleep 3
# hadoop fs -test -e /newread/logbak/novel/$logdate/access/
#done
Delete
Clean
LoadHive
# 插入压缩表read.log_access
#sh /home/hdfs/hadoop-hdfs/personas/clean/read_log_access.sh ${date}
}
main
clean/src/main/resources/log4j.properties 0 → 100644
View file @ 265b180a
log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
clean/src/test/java/com/novel/test/Test.java 0 → 100644
View file @ 265b180a
package com.novel.test;
import com.novel.clean.util.CleanUtil;
import java.util.Map;
public class Test {
public static void main(String[] args) {
String reqParam="{\"sign\":\"985d1ecef2eabd7917c6b4369e15905c\",\"fileType\":\"1\",\"time\":\"1570766944214\",\"is_ad_version\":\"1\",\"source\":\"dialog2\",\"readChapterNum\":\"1\",\"bookId\":\"4198300\",\"chapterNum\":\"2\",\"userPath\":\"dialog,dialog2,C2\"}";
CleanUtil cleanUtil =new CleanUtil();
Map<String, String> book=cleanUtil.getBook(reqParam);
//System.out.println(book.get("bookId"));
//System.out.println(book.get("chapterNum"));
String ua="Mozilla/5.0 (Linux; Android 7.1.1; OD103 Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36\t-999";
String s1 = ua.split(";")[1].split(" ")[2];
System.out.print(s1);
}
}
clean/target/classes/api.json 0 → 100644
View file @ 265b180a
{"activity_bargain":{"action":"activity_bargain","detail":{"activity_bargain":{"action_mark":"0","detail_re":"/api/activity/bargain"}},"re":"/api/activity/bargain"},"activity_entry":{"action":"activity_entry","detail":{"activity_entry":{"action_mark":"0","detail_re":"/api/activity/entry"}},"re":"/api/activity/entry"},"activity_get_vip_card":{"action":"activity_get_vip_card","detail":{"activity_get_vip_card":{"action_mark":"0","detail_re":"/api/activity/get_vip_card"}},"re":"/api/activity/get_vip_card"},"activity_intro":{"action":"activity_intro","detail":{"activity_intro":{"action_mark":"0","detail_re":"/api/activity/intro"}},"re":"/api/activity/intro"},"activity_invitation":{"action":"activity_invitation","detail":{"activity_invitation":{"action_mark":"0","detail_re":"/api/activity/invitation"}},"re":"/api/activity/invitation"},"ad_adInfoList":{"action":"ad_adInfoList","detail":{"ad_adInfoList":{"action_mark":"0","detail_re":"/api/ad/adInfoList"}},"re":"/api/ad/adInfoList"},"book_flowInfo":{"action":"book_flowInfo","detail":{"book_flowInfo":{"action_mark":"0","detail_re":"/api/book/flowInfo"}},"re":"/api/book/flowInfo"},"bookcover_bookcoverInfo":{"action":"bookcover_bookcoverInfo","detail":{"bookcover_bookcoverInfo":{"action_mark":"1","detail_re":"/api/bookcover/bookcoverInfo"}},"re":"/api/bookcover/bookcoverInfo"},"bookcover_comment":{"action":"bookcover_comment","detail":{"bookcover_comment":{"action_mark":"1","detail_re":"/api/bookcover/comment"}},"re":"/api/bookcover/comment"},"bookcover_commentList":{"action":"bookcover_commentList","detail":{"bookcover_commentList":{"action_mark":"1","detail_re":"/api/bookcover/commentList"}},"re":"/api/bookcover/commentList"},"bookcover_recommend":{"action":"bookcover_recommend","detail":{"bookcover_recommend":{"action_mark":"1","detail_re":"/api/bookcover/recommend"}},"re":"/api/bookcover/recommend"},"bookcover_reply":{"action":"bookcover_reply","detail":{"bookcover_reply":{"action_mark":"0","detail_re":"/api/bookcover/reply"}},"re":"/api/bookcover/reply"},"bookcover_replyList":{"action":"bookcover_replyList","detail":{"bookcover_replyList":{"action_mark":"0","detail_re":"/api/bookcover/replyList"}},"re":"/api/bookcover/replyList"},"bookcover_shareInfo":{"action":"bookcover_shareInfo","detail":{"bookcover_shareInfo":{"action_mark":"1","detail_re":"/api/bookcover/shareInfo"}},"re":"/api/bookcover/shareInfo"},"bookshelf_addBookshelf":{"action":"bookshelf_addBookshelf","detail":{"bookshelf_addBookshelf":{"action_mark":"1","detail_re":"/api/bookshelf/addBookshelf"}},"re":"/api/bookshelf/addBookshelf"},"bookshelf_bookUpdateInfo":{"action":"bookshelf_bookUpdateInfo","detail":{"bookshelf_bookUpdateInfo":{"action_mark":"2","detail_re":"/api/bookshelf/bookUpdateInfo"}},"re":"/api/bookshelf/bookUpdateInfo"},"bookshelf_delete":{"action":"bookshelf_delete","detail":{"bookshelf_delete":{"action_mark":"0","detail_re":"/api/bookshelf/delete"}},"re":"/api/bookshelf/delete"},"bookshelf_deleteHistory":{"action":"bookshelf_deleteHistory","detail":{"bookshelf_deleteHistory":{"action_mark":"0","detail_re":"/api/bookshelf/deleteHistory"}},"re":"/api/bookshelf/deleteHistory"},"bookshelf_history":{"action":"bookshelf_history","detail":{"bookshelf_history":{"action_mark":"0","detail_re":"/api/bookshelf/history"}},"re":"/api/bookshelf/history"},"bookshelf_noticeList":{"action":"bookshelf_noticeList","detail":{"bookshelf_noticeList":{"action_mark":"0","detail_re":"/api/bookshelf/noticeList"}},"re":"/api/bookshelf/noticeList"},"bookshelf_recommend":{"action":"bookshelf_recommend","detail":{"bookshelf_recommend":{"action_mark":"0","detail_re":"/api/bookshelf/recommend"}},"re":"/api/bookshelf/recommend"},"bookshelf_recommendBook":{"action":"bookshelf_recommendBook","detail":{"bookshelf_recommendBook":{"action_mark":"0","detail_re":"/api/bookshelf/recommendBook"}},"re":"/api/bookshelf/recommendBook"},"bookshelf_recommendBookList":{"action":"bookshelf_recommendBookList","detail":{"bookshelf_recommendBookList":{"action_mark":"0","detail_re":"/api/bookshelf/recommendBookList"}},"re":"/api/bookshelf/recommendBookList"},"bookshelf_update":{"action":"bookshelf_update","detail":{"bookshelf_update":{"action_mark":"1","detail_re":"/api/bookshelf/update"}},"re":"/api/bookshelf/update"},"bookstore_activity_list":{"action":"bookstore_activity_list","detail":{"bookstore_activity_list":{"action_mark":"0","detail_re":"/api/bookstore/activity_list"}},"re":"/api/bookstore/activity_list"},"bookstore_associativeSearch":{"action":"bookstore_associativeSearch","detail":{"bookstore_associativeSearch":{"action_mark":"0","detail_re":"/api/bookstore/associativeSearch"}},"re":"/api/bookstore/associativeSearch"},"bookstore_bookInfoList":{"action":"bookstore_bookInfoList","detail":{"bookstore_bookInfoList":{"action_mark":"0","detail_re":"/api/bookstore/bookInfoList"}},"re":"/api/bookstore/bookInfoList"},"bookstore_bookListChange":{"action":"bookstore_bookListChange","detail":{"bookstore_bookListChange":{"action_mark":"0","detail_re":"/api/bookstore/bookListChange"}},"re":"/api/bookstore/bookListChange"},"bookstore_channelInfo":{"action":"bookstore_channelInfo","detail":{"bookstore_channelInfo":{"action_mark":"0","detail_re":"/api/bookstore/channelInfo"}},"re":"/api/bookstore/channelInfo"},"bookstore_channelList":{"action":"bookstore_channelList","detail":{"bookstore_channelList":{"action_mark":"0","detail_re":"/api/bookstore/channelList"}},"re":"/api/bookstore/channelList"},"bookstore_classifyList":{"action":"bookstore_classifyList","detail":{"bookstore_classifyList":{"action_mark":"0","detail_re":"/api/bookstore/classifyList"}},"re":"/api/bookstore/classifyList"},"bookstore_editorList":{"action":"bookstore_editorList","detail":{"bookstore_editorList":{"action_mark":"0","detail_re":"/api/bookstore/editorList"}},"re":"/api/bookstore/editorList"},"bookstore_editorRecommendList":{"action":"bookstore_editorRecommendList","detail":{"bookstore_editorRecommendList":{"action_mark":"0","detail_re":"/api/bookstore/editorRecommendList"}},"re":"/api/bookstore/editorRecommendList"},"bookstore_hotSearch":{"action":"bookstore_hotSearch","detail":{"bookstore_hotSearch":{"action_mark":"0","detail_re":"/api/bookstore/hotSearch"}},"re":"/api/bookstore/hotSearch"},"bookstore_ranking":{"action":"bookstore_ranking","detail":{"bookstore_ranking":{"action_mark":"0","detail_re":"/api/bookstore/ranking"}},"re":"/api/bookstore/ranking"},"bookstore_search":{"action":"bookstore_search","detail":{"bookstore_search":{"action_mark":"0","detail_re":"/api/bookstore/search"}},"re":"/api/bookstore/search"},"operation_eventDialogList":{"action":"operation_eventDialogList","detail":{"operation_eventDialogList":{"action_mark":"0","detail_re":"/api/operation/eventDialogList"}},"re":"/api/operation/eventDialogList"},"operation_getBooksForNewUser":{"action":"operation_getBooksForNewUser","detail":{"operation_getBooksForNewUser":{"action_mark":"0","detail_re":"/api/operation/getBooksForNewUser"}},"re":"/api/operation/getBooksForNewUser"},"operation_getComerGiftInfo":{"action":"operation_getComerGiftInfo","detail":{"operation_getComerGiftInfo":{"action_mark":"0","detail_re":"/api/operation/getComerGiftInfo"}},"re":"/api/operation/getComerGiftInfo"},"operation_getOperatingActivitiesList":{"action":"operation_getOperatingActivitiesList","detail":{"operation_getOperatingActivitiesList":{"action_mark":"0","detail_re":"/api/operation/getOperatingActivitiesList"}},"re":"/api/operation/getOperatingActivitiesList"},"operation_getScrollsForCommand":{"action":"operation_getScrollsForCommand","detail":{"operation_getScrollsForCommand":{"action_mark":"0","detail_re":"/api/operation/getScrollsForCommand"}},"re":"/api/operation/getScrollsForCommand"},"operation_incentiveVideoGift":{"action":"operation_incentiveVideoGift","detail":{"operation_incentiveVideoGift":{"action_mark":"0","detail_re":"/api/operation/incentiveVideoGift"}},"re":"/api/operation/incentiveVideoGift"},"operation_interactAd":{"action":"operation_interactAd","detail":{"operation_interactAd":{"action_mark":"0","detail_re":"/api/operation/interactAd"}},"re":"/api/operation/interactAd"},"operation_privilegesForNewUser":{"action":"operation_privilegesForNewUser","detail":{"operation_privilegesForNewUser":{"action_mark":"0","detail_re":"/api/operation/privilegesForNewUser"}},"re":"/api/operation/privilegesForNewUser"},"operation_statistics":{"action":"operation_statistics","detail":{"operation_statistics":{"action_mark":"0","detail_re":"/api/operation/statistics"}},"re":"/api/operation/statistics"},"other_actionLog":{"action":"other_actionLog","detail":{"other_actionLog":{"action_mark":"0","detail_re":"/api/other/actionLog"}},"re":"/api/other/actionLog"},"other_config":{"action":"other_config","detail":{"other_config":{"action_mark":"0","detail_re":"/api/other/config"}},"re":"/api/other/config"},"other_newUserActionLog":{"action":"other_newUserActionLog","detail":{"other_newUserActionLog":{"action_mark":"0","detail_re":"/api/other/newUserActionLog"}},"re":"/api/other/newUserActionLog"},"other_phoneInfo":{"action":"other_phoneInfo","detail":{"other_phoneInfo":{"action_mark":"0","detail_re":"/api/other/phoneInfo"}},"re":"/api/other/phoneInfo"},"other_portrayal":{"action":"other_portrayal","detail":{"other_portrayal":{"action_mark":"0","detail_re":"/api/other/portrayal"}},"re":"/api/other/portrayal"},"other_readFeedback":{"action":"other_readFeedback","detail":{"other_readFeedback":{"action_mark":"2","detail_re":"/api/other/readFeedback"}},"re":"/api/other/readFeedback"},"read_adChapter":{"action":"read_adChapter","detail":{"read_adChapter":{"action_mark":"0","detail_re":"/api/read/adChapter"}},"re":"/api/read/adChapter"},"read_addHistory":{"action":"read_addHistory","detail":{"read_addHistory":{"action_mark":"2","detail_re":"/api/read/addHistory"}},"re":"/api/read/addHistory"},"read_buyBatchChapter":{"action":"read_buyBatchChapter","detail":{"read_buyBatchChapter":{"action_mark":"2","detail_re":"/api/read/buyBatchChapter"}},"re":"/api/read/buyBatchChapter"},"read_buyBook":{"action":"read_buyBook","detail":{"read_buyBook":{"action_mark":"1","detail_re":"/api/read/buyBook"}},"re":"/api/read/buyBook"},"read_buyChapter":{"action":"read_buyChapter","detail":{"read_buyChapter":{"action_mark":"2","detail_re":"/api/read/buyChapter"}},"re":"/api/read/buyChapter"},"read_cacheChapterInfo":{"action":"read","detail":{"read_cacheChapterInfo":{"action_mark":"2","detail_re":"/api/read/cacheChapterInfo"}},"re":"/api/read/cacheChapterInfo"},"read_chapterInfo":{"action":"read","detail":{"read_chapterInfo":{"action_mark":"2","detail_re":"/api/read/chapterInfo"}},"re":"/api/read/chapterInfo"},"read_directory":{"action":"read_directory","detail":{"read_directory":{"action_mark":"0","detail_re":"/api/read/directory"}},"re":"/api/read/directory"},"read_endChangeMoreInfo":{"action":"read_endChangeMoreInfo","detail":{"read_endChangeMoreInfo":{"action_mark":"1","detail_re":"/api/read/endChangeMoreInfo"}},"re":"/api/read/endChangeMoreInfo"},"read_endInfo":{"action":"read_endInfo","detail":{"read_endInfo":{"action_mark":"1","detail_re":"/api/read/endInfo"}},"re":"/api/read/endInfo"},"read_getBuyChapterInfo":{"action":"read_getBuyChapterInfo","detail":{"read_getBuyChapterInfo":{"action_mark":"1","detail_re":"/api/read/getBuyChapterInfo"}},"re":"/api/read/getBuyChapterInfo"},"read_getChapterPayInfo":{"action":"read_getChapterPayInfo","detail":{"read_getChapterPayInfo":{"action_mark":"0","detail_re":"/api/read/getChapterPayInfo"}},"re":"/api/read/getChapterPayInfo"},"read_getHistory":{"action":"read_getHistory","detail":{"read_getHistory":{"action_mark":"1","detail_re":"/api/read/getHistory"}},"re":"/api/read/getHistory"},"read_jumpChapterInfo":{"action":"read_jumpChapterInfo","detail":{"read_jumpChapterInfo":{"action_mark":"1","detail_re":"/api/read/jumpChapterInfo"}},"re":"/api/read/jumpChapterInfo"},"read_like":{"action":"read_like","detail":{"read_like":{"action_mark":"0","detail_re":"/api/read/like"}},"re":"/api/read/like"},"read_reward":{"action":"read_reward","detail":{"read_reward":{"action_mark":"1","detail_re":"/api/read/reward"}},"re":"/api/read/reward"},"read_rewardInfo":{"action":"read_rewardInfo","detail":{"read_rewardInfo":{"action_mark":"1","detail_re":"/api/read/rewardInfo"}},"re":"/api/read/rewardInfo"},"read_urge":{"action":"read_urge","detail":{"read_urge":{"action_mark":"2","detail_re":"/api/read/urge"}},"re":"/api/read/urge"},"recharge_aliPayInfo":{"action":"recharge_aliPayInfo","detail":{"recharge_aliPayInfo":{"action_mark":"0","detail_re":"/api/recharge/aliPayInfo"}},"re":"/api/recharge/aliPayInfo"},"recharge_aliPayInfoMonthly":{"action":"recharge_aliPayInfoMonthly","detail":{"recharge_aliPayInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/aliPayInfoMonthly"}},"re":"/api/recharge/aliPayInfoMonthly"},"recharge_cancelAliPayInfo":{"action":"recharge_cancelAliPayInfo","detail":{"recharge_cancelAliPayInfo":{"action_mark":"0","detail_re":"/api/recharge/cancelAliPayInfo"}},"re":"/api/recharge/cancelAliPayInfo"},"recharge_cancelWeChatPayInfo":{"action":"recharge_cancelWeChatPayInfo","detail":{"recharge_cancelWeChatPayInfo":{"action_mark":"0","detail_re":"/api/recharge/cancelWeChatPayInfo"}},"re":"/api/recharge/cancelWeChatPayInfo"},"recharge_changeInfo":{"action":"recharge_changeInfo","detail":{"recharge_changeInfo":{"action_mark":"0","detail_re":"/api/recharge/changeInfo"}},"re":"/api/recharge/changeInfo"},"recharge_changeInfoMonthly":{"action":"recharge_changeInfoMonthly","detail":{"recharge_changeInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/changeInfoMonthly"}},"re":"/api/recharge/changeInfoMonthly"},"recharge_iOS_changeInfoMonthly":{"action":"recharge_iOS_changeInfoMonthly","detail":{"recharge_iOS_changeInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/iOS/changeInfoMonthly"}},"re":"/api/recharge/iOS/changeInfoMonthly"},"recharge_iOS_iapVerifyMonthly":{"action":"recharge_iOS_iapVerifyMonthly","detail":{"recharge_iOS_iapVerifyMonthly":{"action_mark":"0","detail_re":"/api/recharge/iOS/iapVerifyMonthly"}},"re":"/api/recharge/iOS/iapVerifyMonthly"},"recharge_iOS_returnDiscount":{"action":"recharge_iOS_returnDiscount","detail":{"recharge_iOS_returnDiscount":{"action_mark":"0","detail_re":"/api/recharge/iOS/returnDiscount"}},"re":"/api/recharge/iOS/returnDiscount"},"recharge_iOSchangeInfo":{"action":"recharge_iOSchangeInfo","detail":{"recharge_iOSchangeInfo":{"action_mark":"0","detail_re":"/api/recharge/iOS/changeInfo"}},"re":"/api/recharge/iOS/changeInfo"},"recharge_iOSiapVerify":{"action":"recharge_iOSiapVerify","detail":{"recharge_iOSiapVerify":{"action_mark":"0","detail_re":"/api/recharge/iOS/iapVerify"}},"re":"/api/recharge/iOS/iapVerify"},"recharge_paynowInfo":{"action":"recharge_paynowInfo","detail":{"recharge_paynowInfo":{"action_mark":"0","detail_re":"/api/recharge/paynowInfo"}},"re":"/api/recharge/paynowInfo"},"recharge_pub_wechat_pay_info":{"action":"recharge_pub_wechat_pay_info","detail":{"recharge_pub_wechat_pay_info":{"action_mark":"0","detail_re":"/api/recharge/pub_wechat_pay_info"}},"re":"/api/recharge/pub_wechat_pay_info"},"recharge_returnDiscount":{"action":"recharge_returnDiscount","detail":{"recharge_returnDiscount":{"action_mark":"0","detail_re":"/api/recharge/returnDiscount"}},"re":"/api/recharge/returnDiscount"},"recharge_weChatPayInfo":{"action":"recharge_weChatPayInfo","detail":{"recharge_weChatPayInfo":{"action_mark":"0","detail_re":"/api/recharge/weChatPayInfo"}},"re":"/api/recharge/weChatPayInfo"},"recharge_weChatPayInfoMonthly":{"action":"recharge_weChatPayInfoMonthly","detail":{"recharge_weChatPayInfoMonthly":{"action_mark":"0","detail_re":"/api/recharge/weChatPayInfoMonthly"}},"re":"/api/recharge/weChatPayInfoMonthly"},"recharge_wechat_h5_pay_info":{"action":"recharge_wechat_h5_pay_info","detail":{"recharge_wechat_h5_pay_info":{"action_mark":"0","detail_re":"/api/recharge/wechat_h5_pay_info"}},"re":"/api/recharge/wechat_h5_pay_info"},"recharge_wechat_h5_pay_result_info":{"action":"recharge_wechat_h5_pay_result_info","detail":{"recharge_wechat_h5_pay_result_info":{"action_mark":"0","detail_re":"/api/recharge/wechat_h5_pay_result_info"}},"re":"/api/recharge/wechat_h5_pay_result_info"},"task_common_config":{"action":"task_common_config","detail":{"task_common_config":{"action_mark":"0","detail_re":"/api/task/v1/common/config"}},"re":"/api/task/v1/common/config\r\n"},"task_common_notice":{"action":"task_common_notice","detail":{"task_common_notice":{"action_mark":"0","detail_re":"/api/task/v1/common/notice"}},"re":"/api/task/v1/common/notice"},"task_common_qiniu_token":{"action":"task_common_qiniu_token","detail":{"task_common_qiniu_token":{"action_mark":"0","detail_re":"/api/task/v1/common/qiniu/token"}},"re":"/api/task/v1/common/qiniu/token"},"task_common_text_config":{"action":"task_common_text_config","detail":{"task_common_text_config":{"action_mark":"0","detail_re":"/api/task/v1/common/text/config"}},"re":"/api/task/v1/common/text/config"},"task_complete":{"action":"task_complete","detail":{"task_complete":{"action_mark":"0","detail_re":"/api/task/complete"}},"re":"/api/task/complete"},"task_getList":{"action":"task_getList","detail":{"task_getList":{"action_mark":"0","detail_re":"/api/task/getList"}},"re":"/api/task/getList"},"task_getTaskInfo":{"action":"task_getTaskInfo","detail":{"task_getTaskInfo":{"action_mark":"0","detail_re":"/api/task/getTaskInfo"}},"re":"/api/task/getTaskInfo"},"task_share_user_bind":{"action":"task_share_user_bind","detail":{"task_share_user_bind":{"action_mark":"0","detail_re":"/api/task/v1/share/user/bind"}},"re":"/api/task/v1/share/user/bind"},"task_share_user_share":{"action":"task_share_user_share","detail":{"task_share_user_share":{"action_mark":"0","detail_re":"/api/task/v1/share/user/share"}},"re":"/api/task/v1/share/user/share"},"task_task_finish":{"action":"task_task_finish","detail":{"task_task_finish":{"action_mark":"0","detail_re":"/api/task/v1/task/finish"}},"re":"/api/task/v1/task/finish"},"task_task_list":{"action":"task_task_list","detail":{"task_task_list":{"action_mark":"0","detail_re":"/api/task/v1/task/list"}},"re":"/api/task/v1/task/list"},"task_wallet_user_info":{"action":"task_wallet_user_info","detail":{"task_wallet_user_info":{"action_mark":"0","detail_re":"/api/task/v1/wallet/user/info"}},"re":"/api/task/v1/wallet/user/info"},"task_wallet_user_records":{"action":"task_wallet_user_records","detail":{"task_wallet_user_records":{"action_mark":"0","detail_re":"/api/task/v1/wallet/user/records"}},"re":"/api/task/v1/wallet/user/records"},"task_wallet_user_share_records":{"action":"task_wallet_user_share_records","detail":{"task_wallet_user_share_records":{"action_mark":"0","detail_re":"/api/task/v1/wallet/user/share/records"}},"re":"/api/task/v1/wallet/user/share/records"},"task_withdraw_list":{"action":"task_withdraw_list","detail":{"task_withdraw_list":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/list"}},"re":"/api/task/v1/withdraw/list"},"task_withdraw_records":{"action":"task_withdraw_records","detail":{"task_withdraw_records":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/records"}},"re":"/api/task/v1/withdraw/records"},"task_withdraw_submit":{"action":"task_withdraw_submit","detail":{"task_withdraw_submit":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/submit"}},"re":"/api/task/v1/withdraw/submit"},"task_withdraw_valid":{"action":"task_withdraw_valid","detail":{"task_withdraw_valid":{"action_mark":"0","detail_re":"/api/task/v1/withdraw/valid"}},"re":"/api/task/v1/withdraw/valid"},"user_Register":{"action":"user_Register","detail":{"user_Register":{"action_mark":"0","detail_re":"/api/user/autoRegister"}},"re":"/api/user/autoRegister"},"user_bindPhoneGiftBagInfo":{"action":"user_bindPhoneGiftBagInfo","detail":{"user_bindPhoneGiftBagInfo":{"action_mark":"0","detail_re":"/api/user/bindPhoneGiftBagInfo"}},"re":"/api/user/bindPhoneGiftBagInfo"},"user_changePhoneNum":{"action":"user_changePhoneNum","detail":{"user_changePhoneNum":{"action_mark":"0","detail_re":"/api/user/changePhoneNum"}},"re":"/api/user/changePhoneNum"},"user_coinList":{"action":"user_coinList","detail":{"user_coinList":{"action_mark":"0","detail_re":"/api/user/coinList"}},"re":"/api/user/coinList"},"user_consumeInfo":{"action":"user_consumeInfo","detail":{"user_consumeInfo":{"action_mark":"0","detail_re":"/api/user/consumeInfo"}},"re":"/api/user/consumeInfo"},"user_feedback":{"action":"user_feedback","detail":{"user_feedback":{"action_mark":"0","detail_re":"/api/feedback"}},"re":"/api/feedback"},"user_getPublicKey":{"action":"user_getPublicKey","detail":{"user_getPublicKey":{"action_mark":"0","detail_re":"/api/getPublicKey"}},"re":"/api/getPublicKey"},"user_h5_thirdLogin":{"action":"user_h5_thirdLogin","detail":{"user_h5_thirdLogin":{"action_mark":"0","detail_re":"/api/user/h5/thirdLogin"}},"re":"/api/user/h5/thirdLogin"},"user_modifyUserInfo":{"action":"user_modifyUserInfo","detail":{"user_modifyUserInfo":{"action_mark":"0","detail_re":"/api/user/modifyUserInfo"}},"re":"/api/user/modifyUserInfo"},"user_msgAuthCode":{"action":"user_msgAuthCode","detail":{"user_msgAuthCode":{"action_mark":"0","detail_re":"/api/user/msgAuthCode"}},"re":"/api/user/msgAuthCode"},"user_openAd":{"action":"user_openAd","detail":{"user_openAd":{"action_mark":"0","detail_re":"/api/openAd"}},"re":"/api/openAd"},"user_phoneNumLogin":{"action":"user_phoneNumLogin","detail":{"user_phoneNumLogin":{"action_mark":"0","detail_re":"/api/user/phoneNumLogin"}},"re":"/api/user/phoneNumLogin"},"user_preference":{"action":"user_preference","detail":{"user_preference":{"action_mark":"0","detail_re":"/api/user/preference"}},"re":"/api/user/preference"},"user_question":{"action":"user_question","detail":{"user_question":{"action_mark":"0","detail_re":"/api/user/question"}},"re":"/api/user/question"},"user_rechargeInfo":{"action":"user_rechargeInfo","detail":{"user_rechargeInfo":{"action_mark":"0","detail_re":"/api/user/rechargeInfo"}},"re":"/api/user/rechargeInfo"},"user_setPreference":{"action":"user_setPreference","detail":{"user_setPreference":{"action_mark":"0","detail_re":"/api/user/setPreference"}},"re":"/api/user/setPreference"},"user_shareResult":{"action":"user_shareResult","detail":{"user_shareResult":{"action_mark":"0","detail_re":"/api/user/shareResult"}},"re":"/api/user/shareResult"},"user_signGift":{"action":"user_signGift","detail":{"user_signGift":{"action_mark":"0","detail_re":"/api/user/signGift"}},"re":"/api/user/signGift"},"user_signIn":{"action":"user_signIn","detail":{"user_signIn":{"action_mark":"0","detail_re":"/api/user/signIn"}},"re":"/api/user/signIn"},"user_signInInfo":{"action":"user_signInInfo","detail":{"user_signInInfo":{"action_mark":"0","detail_re":"/api/user/signInInfo"}},"re":"/api/user/signInInfo"},"user_thirdLogin":{"action":"user_thirdLogin","detail":{"user_thirdLogin":{"action_mark":"0","detail_re":"/api/user/thirdLogin"}},"re":"/api/user/thirdLogin"},"user_unbindThirdLogin":{"action":"user_unbindThirdLogin","detail":{"user_unbindThirdLogin":{"action_mark":"0","detail_re":"/api/user/unbindThirdLogin"}},"re":"/api/user/unbindThirdLogin"},"user_update":{"action":"user_update","detail":{"user_update":{"action_mark":"0","detail_re":"/api/update"}},"re":"/api/update"},"user_userInfo":{"action":"user_userInfo","detail":{"user_userInfo":{"action_mark":"0","detail_re":"/api/user/userInfo"}},"re":"/api/user/userInfo"}}
clean/target/classes/clean_api.py 0 → 100644
View file @ 265b180a
#!/usr/bin/env python
# -* - coding: UTF-8 -* -
#name : by demon
#time : 2017-05-24
import sys
import re
import time
import os
import datetime
import socket,json
import urllib
import traceback
from struct import pack, unpack
reload(sys)
sys.setdefaultencoding('utf-8')
class clearurl():
#@staticmethod
def __init__(self, re_path):
self.re_path = re_path
self.re_mess = os.popen('hadoop fs -cat %s' %re_path).read()
self.dict_re=eval(self.re_mess)
def clear_api(self,requrlstr,reqParam):
dict_re = self.dict_re
#定义变量
action="-999"
action_detail="-999"
bookid="-999"
chapterNum="-999"
querymessage="-999"
requrl = requrlstr.split("?")[0]
for i in dict_re.keys():
if 're' not in dict_re[i]:
continue
patterni = re.compile(r'^%s' % dict_re[i]['re'])
break_flag = False
if patterni.match(requrl):
for n in dict_re[i]['detail'].keys():
patternn = re.compile(r'^%s' % dict_re[i]['detail'][n]['detail_re'])
if patternn.match(requrl):
break_flag = True
action = dict_re[i]['action']
action_detail = n
action_mark = dict_re[i]['detail'][n]['action_mark']
try:
if action == 'search':
querystr = requrlstr.split('query=')[1].split('&')[0]
querymessage = urllib.unquote(querystr)
else:
querymessage = '-999'
except:
querymessage='-999'
#print action_mark,reqParam
try:
if action_mark == "0":
bookid = "-999"
chapterNum = "-999"
elif action_mark == "1":
reqjson=json.loads(reqParam)
if 'bookIds' in reqParam:
bookid=reqjson['bookIds']
elif 'bookId' in reqParam:
bookid=reqjson['bookId']
else:
bookid='-999'
chapterNum="-999"
elif action_mark == "2":
if reqParam !='':
reqjson = json.loads(reqParam)
if 'bookIds' in reqParam:
bookid=reqjson['bookIds']
elif 'bookId' in reqParam:
bookid=reqjson['bookId']
else:
bookid='-999'
if 'chapterNum' in reqParam:
chapterNum = reqjson['chapterNum']
else:
chapterNum="-999"
else:
bookid = "-999"
chapterNum = "-999"
except:
bookid = "-999"
chapterNum = "-999"
break
if break_flag:
break
return action, action_detail, bookid, chapterNum,querymessage
def clear_refer(self,refer,reqParam,platform):
# print refer
try:
refer_com = refer.split("/")[2]
pattern_com = re.compile(r'[a-z]+.yc.ifeng.com')
if pattern_com.match(refer_com):
refer_url = refer.split(".com")[1].split("?")[0]
#print refer_url
try:
if platform == "api":
referaction, referaction_detail, referbookid, referchapterNum,querymessage = self.clear_api(refer_url,reqParam)
except:
referaction = "-999"
else:
referaction = "-999"
except:
refer_com = "-999"
referaction = "-999"
return refer_com, referaction
class IPInfo():
'''QQWry.Dat数据库查询功能集合
'''
def __init__(self, dbname):
''' 初始化类，读取数据库内容为一个字符串，
通过开始8字节确定数据库的索引信息'''
self.dbname = dbname
# f = file(dbname, 'r')
# Demon注：在Windows下用'r'会有问题，会把\r\n转换成\n
# 详见http://demon.tw/programming/python-open-mode.html
# 还有Python文档中不提倡用file函数来打开文件，推荐用open
f = open(dbname, 'rb')
self.img = f.read()
f.close()
# QQWry.Dat文件的开始8字节是索引信息,前4字节是开始索引的偏移值，
# 后4字节是结束索引的偏移值。
# (self.firstIndex, self.lastIndex) = unpack('II', self.img[:8])
# Demon注：unpack默认使用的endian是和机器有关的
# Intel x86和AMD64(x86-64)是little-endian
# Motorola 68000和PowerPC G5是big-endian
# 而纯真数据库全部采用了little-endian字节序
# 所以在某些big-endian的机器上原代码会出错
(self.firstIndex, self.lastIndex) = unpack('<II', self.img[:8])
# 每条索引长7字节，这里得到索引总个数
self.indexCount = (self.lastIndex - self.firstIndex) / 7 + 1
def getString(self, offset=0):
''' 读取字符串信息，包括"国家"信息和"地区"信息
QQWry.Dat的记录区每条信息都是一个以'\0'结尾的字符串'''
o2 = self.img.find('\0', offset)
# return self.img[offset:o2]
# 有可能只有国家信息没有地区信息，
gb2312_str = self.img[offset:o2]
try:
utf8_str = unicode(gb2312_str, 'gb2312').encode('utf-8')
except:
return '未知'
return utf8_str
def getLong3(self, offset=0):
'''QQWry.Dat中的偏移记录都是3字节，本函数取得3字节的偏移量的常规表示
QQWry.Dat使用"字符串"存储这些值'''
s = self.img[offset: offset + 3]
s += '\0'
# unpack用一个'I'作为format，后面的字符串必须是4字节
# return unpack('I', s)[0]
# Demon注：和上面一样，强制使用little-endian
return unpack('<I', s)[0]
def getAreaAddr(self, offset=0):
''' 通过给出偏移值，取得区域信息字符串，'''
byte = ord(self.img[offset])
if byte == 1 or byte == 2:
# 第一个字节为1或者2时，取得2-4字节作为一个偏移量调用自己
p = self.getLong3(offset + 1)
return self.getAreaAddr(p)
else:
return self.getString(offset)
def getAddr(self, offset, ip=0):
img = self.img
o = offset
byte = ord(img[o])
if byte == 1:
# 重定向模式1
# [IP][0x01][国家和地区信息的绝对偏移地址]
# 使用接下来的3字节作为偏移量调用字节取得信息
return self.getAddr(self.getLong3(o + 1))
if byte == 2:
# 重定向模式2
# [IP][0x02][国家信息的绝对偏移][地区信息字符串]
# 使用国家信息偏移量调用自己取得字符串信息
cArea = self.getAreaAddr(self.getLong3(o + 1))
o += 4
# 跳过前4字节取字符串作为地区信息
aArea = self.getAreaAddr(o)
return (cArea, aArea)
if byte != 1 and byte != 2:
# 最简单的IP记录形式，[IP][国家信息][地区信息]
# 重定向模式1有种情况就是偏移量指向包含国家和地区信息两个字符串
# 即偏移量指向的第一个字节不是1或2,就使用这里的分支
# 简单地说：取连续取两个字符串！
cArea = self.getString(o)
# o += len(cArea) + 1
# 我们已经修改cArea为utf-8字符编码了，len取得的长度会有变，
# 用下面方法得到offset
o = self.img.find('\0', o) + 1
aArea = self.getString(o)
return (cArea, aArea)
def find(self, ip, l, r):
''' 使用二分法查找网络字节编码的IP地址的索引记录'''
if r - l <= 1:
return l
m = (l + r) / 2
o = self.firstIndex + m * 7
# new_ip = unpack('I', self.img[o: o+4])[0]
# Demon注：和上面一样，强制使用little-endian
new_ip = unpack('<I', self.img[o: o + 4])[0]
if ip <= new_ip:
return self.find(ip, l, m)
else:
return self.find(ip, m, r)
def getIPAddr(self, ip):
''' 调用其他函数，取得信息！'''
# 使用网络字节编码IP地址
ip = unpack('!I', socket.inet_aton(ip))[0]
# 使用 self.find 函数查找ip的索引偏移
i = self.find(ip, 0, self.indexCount - 1)
# 得到索引记录
o = self.firstIndex + i * 7
# 索引记录格式是： 前4字节IP信息+3字节指向IP记录信息的偏移量
# 这里就是使用后3字节作为偏移量得到其常规表示（QQWry.Dat用字符串表示值）
o2 = self.getLong3(o + 4)
# IP记录偏移值+4可以丢弃前4字节的IP地址信息。
(c, a) = self.getAddr(o2 + 4)
return (c, a)
def output(self, first, last):
for i in range(first, last):
o = self.firstIndex + i * 7
ip = socket.inet_ntoa(pack('!I', unpack('I', self.img[o:o + 4])[0]))
offset = self.getLong3(o + 4)
(c, a) = self.getAddr(offset + 4)
print "%s %d %s/%s" % (ip, offset, c, a)
# def getIP(self,address):
# i = IPInfo(self.dbname)
# (c, a) = i.getIPAddr(address)
# Demon注：如果是在Windows命令行中运行把编码转回gb2312以避免乱码
# if sys.platform == 'win32':
# c = unicode(c, 'utf-8').encode('gb2312')
# a = unicode(a, 'utf-8').encode('gb2312')
return c
def read_input(file):
for line in file:
yield line.strip()
def main():
platform = "api"
#加载清洗url脚本
clear = clearurl('/newread/tool/clean/dictre/%s.re' %platform)
#加载IPSeek
if os.path.isfile('/tmp/QQWry.Dat'):
pass
else:
os.system('hadoop fs -get /newread/tool/clean/QQWry.Dat /tmp/')
#加载IPSeek
P = IPInfo('/tmp/QQWry.Dat')
#清洗程序开始
data = read_input(sys.stdin)
for line in data:
try:
list = line.split("] [")
#切割日志各字段
logversion = list[0].split("[")[1]
projectname = list[1]
serverip = list[2]
reqbegintime = list[3]
reqendtime = list[4]
reqspendtime = list[5]
reqip = list[6]
requrlstr = list[7]
#过滤下载日志 add by zhaopf 2018-12-06
if(('/api/download/' in requrlstr) or ('/api/other/jrttBack' in requrlstr) or ('/kuaishou/notify' in requrlstr) or ('/api/getPublicKey' in requrlstr)):
continue
refer = list[8]
method = list[9]
httpcode = list[10]
userid = list[11]
sessionid = list[12]
platformid = list[13]
if( 'type=bbs' in requrlstr):
if platformid == '1':
platformid=100
elif platformid == '2':
platformid=200
appversion = list[14]
channelid = list[15]
bookrecommend= list[16]
payType = list[17]
paynum = list[18]
rechargetype = list[19]
payamount = list[20]
ssid = list[21]
isnewuser = list[22]
ischannelnewuser = list[23]
readwordstype = list[24]
readwords = list[25]
mobile = list[26]
uaplatform = list[27]
uastat = list[28]
apicode = list[29]
xClient = list[30]
ua = list[31]
if('JianKongBao' in ua ):
continue
#print(list[33])
reqParam = list[32]
processParam = list[33]
#clear refer 定义来源网址以及referaction_detail
refer_com, referaction_detail = clear.clear_refer(refer,reqParam,platform)
#将用户IP转化为省份以及地区
try:
UserIp = reqip.split(',')[0]
(country, detail_area) = P.getIPAddr(UserIp)
except:
country="-999"
detail_area="-999"
#调用clear方法，将URL转化为action等
#定义变量
action="-999"
action_detail="-999"
bookid="-999"
chapterNum="-999"
querymessage="-999"
#try:
if platform =="api":
action, action_detail, bookid, chapterNum ,querymessage = clear.clear_api(requrlstr, reqParam)
#except:
# action = "-999"
# action_detail = "-999"
# bookid = "-999"
# chapterNum = "-999"
# querymessage = '-999'
#切分Xclient取出相应的数据
try:
sdk = xClient.split('sv=')[1].split(';')[0]
except:
sdk = "-999"
try:
screensize = xClient.split('ss=')[1].split(';')[0]
except:
screensize = "-999"
try:
type = xClient.split('pm=')[1].split(';')[0]
type=urllib.unquote(type)
except:
type = "-999"
try:
imsi = xClient.split('imsi=')[1].split(';')[0]
except:
imsi = "-999"
try:
version = xClient.split('version=')[1].split(';')[0]
if version is not None and version.find('.') > 0:
#print(version)
first_index = version.find('.') + 1
#print(first_index)
last_index = version.rfind('.')
#print(last_index)
appversion2 = version[first_index:last_index]
#print(appversion2)
except:
version = "-999"
appversion2 = "-999"
try:
mac = xClient.split('mac=')[1].split(';')[0]
except:
mac = "-999"
try:
device_id = xClient.split('dID=')[1].split(';')[0]
except:
device_id = "-999"
rn = "-999"
tdcn = "-999"
try:
imei = xClient.split('imei=')[1].split(';')[0]
except:
imei = '-999'
try:
idfa = xClient.split('idfa=')[1].split(';')[0]
except:
idfa = '-999'
#定义浏览器
endbrand = "-999"
browserbrand = "-999"
browserversion = "-999"
primaryurlpath = "-999"
others = querymessage
cellid = "-999"
projectType="-999"
print '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t' %(logversion,projectname,serverip,reqbegintime,reqendtime,reqspendtime,reqip,country,detail_area,requrlstr,action,action_detail,bookid,chapterNum,refer,refer_com,referaction_detail,method,httpcode,userid,sessionid,platformid,appversion,channelid,bookrecommend,payType,paynum,rechargetype,payamount,ssid,isnewuser,ischannelnewuser,readwordstype,readwords,mobile,uaplatform,uastat,apicode,imei,sdk,screensize,type,imsi,cellid,version,device_id,mac,endbrand,browserbrand,browserversion,ua,primaryurlpath,reqParam,others,processParam,projectType,idfa,appversion2)
except Exception, e:
exc_type, exc_obj, exc_tb = sys.exc_info();
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
#print("line="+line)
#print(exc_type, fname, exc_tb.tb_lineno)
#traceback.print_exc()
continue
if __name__ == "__main__":
main()
clean/target/classes/clean_api.sh 0 → 100644
View file @ 265b180a
#!/bin/sh
#time 2017-06-07
################# 初始化变量#########################
hadoop="/usr/bin/hadoop";
hdfs="/usr/bin/hdfs";
cleanpath="/home/hdfs/hadoop-hdfs/personas/clean";
if [ "$1" != "" ];then
date=`date -d "$1" +%Y-%m-%d`;
logdate=`date -d "$1" +%Y%m%d`;
else
date=`date -d "-1 day" +%Y-%m-%d`;
logdate=`date -d "-1 day" +%Y%m%d`;
fi
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_android.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_android_lbb.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_android_kyy.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_ios.sh $date
/bin/bash /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api_h5.sh $date
clean/target/classes/clean_api_android.sh 0 → 100644
View file @ 265b180a
#!/bin/sh
#time 2017-06-07
################# 初始化变量#########################
source /etc/bi.profile
hadoop="/usr/bin/hadoop";
hdfs="/usr/bin/hdfs";
cleanpath="/home/hdfs/hadoop-hdfs/personas/clean";
if [ "$1" != "" ];then
date=`date -d "$1" +%Y-%m-%d`;
logdate=`date -d "$1" +%Y%m%d`;
else
date=`date -d "-1 day" +%Y-%m-%d`;
logdate=`date -d "-1 day" +%Y%m%d`;
fi
year=${date:0:4}
month=${date:4:2}
export MYSQL_BI_IP=10.90.8.32
export MYSQL_BI_USER=readtongji
export MYSQL_BI_PWD=uZpvSuDuEiEOwCtk21on
export MYSQL_BI_DB=db_bi
export MYSQL_BI_PORT=3306
function Delete(){
hadoop fs -rm -r -f /newread/read/log_access_txt/platform=android/day=${date}/
}
function Clean(){
cd ${cleanpath}
mark=`hadoop fs -ls /newread/logflag/android/${logdate}/*.lock | wc -l`
title="android日志检查"
content="${date}android日志上传成功"
content1="${date}android日志上传失败"
flag=0
if [ $mark -eq 1 ];then
/bin/python ${alert_path}/alert_hive.py $title $content "android"
flag=1
else
/bin/python ${alert_path}/alert_hive.py $title $content1 "android"
fi
while [ $mark -eq 0 ];do
sleep 10
mark=`hadoop fs -ls /newread/logflag/android/${logdate}/*.lock | wc -l`
done
echo ${mark}
if [ $mark -eq 1 -a $flag -eq 0 ];then
/bin/python ${alert_path}/alert_hive.py $title $content "android"
elif [ $mark -eq 0 -a $flag -eq 0 ];then
/bin/python ${alert_path}/alert_hive.py $title $content1 "android"
fi
hadoop jar /usr/hdp/2.6.5.0-292/hadoop-mapreduce/hadoop-streaming.jar -input /newread/logbak/android/${logdate}/*-access-*.log -output /newread/read/log_access_txt/platform=android/day=${date} -file /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py -mapper "/home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py" -jobconf mapred.job.name="job_acclog_last_and"
#hadoop jar /usr/hdp/2.6.5.0-292/hadoop-mapreduce/hadoop-streaming.jar -inputformat com.hadoop.mapred.DeprecatedLzoTextInputFormat -input /newread/logbak/android/${logdate}/*-access-*.log.lzo -output /newread/read/log_access_txt/platform=android/day=${date} -file /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py -mapper "/home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py" -jobconf mapred.job.name="job_acclog_last_and" -jobconf mapred.output.compress=true -jobconf mapred.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec
#check_result "clean_acclog_android"
}
function LoadHive(){
hive <<EOF
use read;
ALTER TABLE log_access_txt DROP IF EXISTS partition (platform='android',day='$date');
alter table log_access_txt add IF NOT EXISTS partition (platform='android',day='$date') location '/newread/read/log_access_txt/platform=android/day=$date';
exit;
EOF
exit;
}
function main(){
#hadoop fs -test -e /newread/logbak/novel/$logdate/access/
#while [ $? -ne 0 ];do
# sleep 3
# hadoop fs -test -e /newread/logbak/novel/$logdate/access/
#done
Delete
Clean
LoadHive
# 插入压缩表read.log_access
#sh /home/hdfs/hadoop-hdfs/personas/clean/read_log_access.sh ${date}
}
main
clean/target/classes/log4j.properties 0 → 100644
View file @ 265b180a
log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
clean/target/maven-archiver/pom.properties 0 → 100644
View file @ 265b180a
#Generated by Maven
#Sun Oct 13 22:37:11 CST 2019
version=1.0
groupId=com.novel.clean
artifactId=clean
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment