1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

######################## BEGIN LICENSE BLOCK ######################## 

# The Original Code is mozilla.org code. 

# 

# The Initial Developer of the Original Code is 

# Netscape Communications Corporation. 

# Portions created by the Initial Developer are Copyright (C) 1998 

# the Initial Developer. All Rights Reserved. 

# 

# Contributor(s): 

# Mark Pilgrim - port to Python 

# 

# This library is free software; you can redistribute it and/or 

# modify it under the terms of the GNU Lesser General Public 

# License as published by the Free Software Foundation; either 

# version 2.1 of the License, or (at your option) any later version. 

# 

# This library is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 

# Lesser General Public License for more details. 

# 

# You should have received a copy of the GNU Lesser General Public 

# License along with this library; if not, write to the Free Software 

# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 

# 02110-1301 USA 

######################### END LICENSE BLOCK ######################### 

 

from .mbcharsetprober import MultiByteCharSetProber 

from .codingstatemachine import CodingStateMachine 

from .chardistribution import SJISDistributionAnalysis 

from .jpcntx import SJISContextAnalysis 

from .mbcssm import SJIS_SM_MODEL 

from .enums import ProbingState, MachineState 

 

 

class SJISProber(MultiByteCharSetProber): 

def __init__(self): 

super(SJISProber, self).__init__() 

self.coding_sm = CodingStateMachine(SJIS_SM_MODEL) 

self.distribution_analyzer = SJISDistributionAnalysis() 

self.context_analyzer = SJISContextAnalysis() 

self.reset() 

 

def reset(self): 

super(SJISProber, self).reset() 

self.context_analyzer.reset() 

 

@property 

def charset_name(self): 

return self.context_analyzer.charset_name 

 

@property 

def language(self): 

return "Japanese" 

 

def feed(self, byte_str): 

for i in range(len(byte_str)): 

coding_state = self.coding_sm.next_state(byte_str[i]) 

if coding_state == MachineState.ERROR: 

self.logger.debug('%s %s prober hit error at byte %s', 

self.charset_name, self.language, i) 

self._state = ProbingState.NOT_ME 

break 

elif coding_state == MachineState.ITS_ME: 

self._state = ProbingState.FOUND_IT 

break 

elif coding_state == MachineState.START: 

char_len = self.coding_sm.get_current_charlen() 

if i == 0: 

self._last_char[1] = byte_str[0] 

self.context_analyzer.feed(self._last_char[2 - char_len:], 

char_len) 

self.distribution_analyzer.feed(self._last_char, char_len) 

else: 

self.context_analyzer.feed(byte_str[i + 1 - char_len:i + 3 

- char_len], char_len) 

self.distribution_analyzer.feed(byte_str[i - 1:i + 1], 

char_len) 

 

self._last_char[0] = byte_str[-1] 

 

if self.state == ProbingState.DETECTING: 

if (self.context_analyzer.got_enough_data() and 

(self.get_confidence() > self.SHORTCUT_THRESHOLD)): 

self._state = ProbingState.FOUND_IT 

 

return self.state 

 

def get_confidence(self): 

context_conf = self.context_analyzer.get_confidence() 

distrib_conf = self.distribution_analyzer.get_confidence() 

return max(context_conf, distrib_conf)